simplify bootstrapping process

allow replacing modules
remove stage0.jl and stage1.jl; replace Base on later stages instead
remove jl_find_file_in_path
save a little space by removing non-exported macros (used only for
generating definitions at load time)

Makefile

@@ -28,13 +28,10 @@ base/build_h.jl: Make.inc
 $(BUILD)/lib/julia/helpdb.jl: doc/helpdb.jl | $(BUILD)/lib/julia
 	@cp $< $@
 
-$(BUILD)/lib/julia/sys0.ji: base/boot.jl src/dump.c base/stage0.jl base/build_h.jl
-	$(QUIET_JULIA) cd base && $(JULIA_EXECUTABLE) -b stage0.jl
-	@rm -f $(BUILD)/lib/julia/sys.ji
-
-# if sys.ji exists, use it to rebuild, otherwise use sys0.ji
-$(BUILD)/lib/julia/sys.ji: VERSION $(BUILD)/lib/julia/sys0.ji base/*.jl $(BUILD)/lib/julia/helpdb.jl
-	$(QUIET_JULIA) cd base && $(JULIA_EXECUTABLE) `test -f $(BUILD)/lib/julia/sys.ji && echo stage1.jl || echo -J $(BUILD)/lib/julia/sys0.ji stage1.jl`
+# use sys.ji if it exists, otherwise run two stages
+$(BUILD)/lib/julia/sys.ji: VERSION base/*.jl $(BUILD)/lib/julia/helpdb.jl
+	$(QUIET_JULIA) cd base && \
+	(test -f $(BUILD)/lib/julia/sys.ji || $(JULIA_EXECUTABLE) -b sysimg.jl) && $(JULIA_EXECUTABLE) sysimg.jl
 
 ifeq ($(OS), WINNT)
 OPENBLASNAME=openblas-r0.1.1

base/sysimg.jl

@@ -1,4 +1,4 @@
-## Base module exports
+module Base
 
 export
     # Module
@@ -199,6 +199,38 @@ export
     @gensym, @eval, @task, @f_str, @thunk, @L_str, @vectorize_1arg,
     @vectorize_2arg, @printf
 
+if false
+    # simple print definitions for debugging. enable these if something
+    # goes wrong during bootstrap before printing code is available.
+    length(a::Array) = arraylen(a)
+    print(x) = print(stdout_stream, x)
+    show(x) = show(stdout_stream, x)
+    write(io, a::Array{Uint8,1}) =
+        ccall(:ios_write, Uint, (Ptr{Void}, Ptr{Void}, Uint),
+              io.ios, a, length(a))
+    print(io, s::Symbol) = ccall(:jl_print_symbol, Void, (Ptr{Void},Any,),
+                                 io.ios, s)
+    print(io, s::ASCIIString) = (write(io, s.data);nothing)
+    print(io, x) = show(io, x)
+    println(io, x) = (print(io, x); print(io, "\n"))
+    show(io, x) = ccall(:jl_show_any, Void, (Any, Any,), io, x)
+    show(io, s::ASCIIString) = (write(io, s.data);nothing)
+    show(io, s::Symbol) = print(io, s)
+    show(io, b::Bool) = print(io, b ? "true" : "false")
+    show(io, n::Int64) = ccall(:jl_print_int64, Void, (Ptr{Void}, Int64,), io, n)
+    show(io, n::Integer)  = show(io, int64(n))
+    print(io, a...) = for x=a; print(io, x); end
+    function show(io, e::Expr)
+        print(io, e.head)
+        print(io, "(")
+        for i=1:arraylen(e.args)
+            show(io, arrayref(e.args,i))
+            print(io, ", ")
+        end
+        print(io, ")\n")
+    end
+end
+
 ## Load essential files and libraries
 
 include("base.jl")
@@ -396,7 +428,7 @@ compile_hint(_jl_eval_user_input, (Expr, Bool))
 compile_hint(print, (Float64,))
 compile_hint(a2t, (Array{Any,1},))
 compile_hint(flush, (IOStream,))
-compile_hint(ref, (Type{String}, ASCIIString, ASCIIString))
+compile_hint(ref, (Type{String}, ASCIIString, ASCIIString, ASCIIString))
 compile_hint(int, (Int,))
 compile_hint(uint, (Uint,))
 compile_hint(_atexit, ())
@@ -413,3 +445,19 @@ compile_hint(IOStream, (ASCIIString, Array{Uint8,1}))
 compile_hint(_jl_mk_tupleref, (SymbolNode, Int))
 compile_hint(_jl_abstract_interpret, (Bool, ObjectIdDict, StaticVarInfo))
 compile_hint(eval_annotate, (LambdaStaticData, ObjectIdDict, StaticVarInfo, ObjectIdDict, Array{Any,1}))
+
+# invoke type inference, running the existing inference code on the new
+# inference code to cache an optimized version of it.
+begin
+    local atypes = (LambdaStaticData, Tuple, (), LambdaStaticData, Bool)
+    local minf = getmethods(typeinf, atypes)
+    typeinf_ext(minf[1][3], atypes, (), minf[1][3])
+end
+
+end # module Base
+
+import Base.*
+
+# create system image file
+ccall(:jl_save_system_image, Void, (Ptr{Uint8},Ptr{Uint8}),
+      "$JULIA_HOME/../lib/julia/sys.ji", "start_image.jl")

src/dump.c

@@ -716,42 +716,18 @@ void jl_save_system_image(char *fname, char *startscriptname)
     ios_t f;
     ios_file(&f, fname, 1, 1, 1, 1);
 
-    if (jl_current_module != jl_base_module &&
-        jl_current_module != jl_main_module) {
-        // set up for stage 1 bootstrap, where the Base module is already
-        // loaded and we are loading an updated copy in a separate module.
+    // orphan old Base module if present
+    jl_base_module = (jl_module_t*)jl_get_global(jl_root_module, jl_symbol("Base"));
 
-        // step 1: set Root.Base = current_module
-        jl_binding_t *b = jl_get_binding_wr(jl_root_module, jl_symbol("Base"));
-        b->value = (jl_value_t*)jl_current_module;
-        assert(b->constp);
+    // remove Main module
+    jl_binding_t *b = jl_get_binding_wr(jl_root_module, jl_symbol("Main"));
+    b->value = NULL; b->constp = 0;
 
-        // step 2: remove Root.current_module
-        b = jl_get_binding_wr(jl_root_module, jl_current_module->name);
-        b->value = NULL; b->constp = 0;
-
-        // step 3: current_module.Base = current_module
-        jl_set_const(jl_current_module, jl_symbol("Base"),
-                     (jl_value_t*)jl_current_module);
-
-        // step 4: remove current_module.current_module
-        b = jl_get_binding_wr(jl_current_module, jl_current_module->name);
-        b->value = NULL; b->constp = 0;
-
-        // step 5: rename current_module to Base
-        jl_current_module->name = jl_symbol("Base");
-
-        // step 6: orphan old Base module
-        jl_base_module = jl_current_module;
-
-        // step 7: remove Main module
-        b = jl_get_binding_wr(jl_root_module, jl_symbol("Main"));
-        b->value = NULL; b->constp = 0;
-    }
-
-    // delete cached slow ASCIIString constructor
+    // delete cached slow ASCIIString constructor if present
     jl_methtable_t *mt = jl_gf_mtable((jl_function_t*)jl_ascii_string_type);
-    if (mt->defs->func->linfo->inferred == jl_false) {
+    jl_array_t *spec = mt->defs->func->linfo->specializations;
+    if (spec->length > 0 &&
+        ((jl_lambda_info_t*)jl_cellref(spec,0))->inferred == jl_false) {
         mt->cache = JL_NULL;
         mt->cache_arg1 = JL_NULL;
         mt->defs->func->linfo->tfunc = (jl_value_t*)jl_null;
@@ -764,7 +740,6 @@ void jl_save_system_image(char *fname, char *startscriptname)
     jl_serialize_value(&f, jl_array_type->env);
 
     jl_serialize_value(&f, jl_root_module);
-    jl_serialize_value(&f, jl_current_module);
 
     jl_serialize_value(&f, idtable_list);
 
@@ -792,7 +767,7 @@ DLLEXPORT
 void jl_restore_system_image(char *fname)
 {
     ios_t f;
-    char *fpath = jl_find_file_in_path(fname);
+    char *fpath = fname;
     if (ios_file(&f, fpath, 1, 0, 0, 0) == NULL) {
         JL_PRINTF(JL_STDERR, "system image file not found\n");
         jl_exit(1);
@@ -807,13 +782,11 @@ void jl_restore_system_image(char *fname)
     jl_array_type->env = jl_deserialize_value(&f);
 
     jl_root_module = (jl_module_t*)jl_deserialize_value(&f);
-    jl_current_module = (jl_module_t*)jl_deserialize_value(&f);
     jl_core_module = (jl_module_t*)jl_get_global(jl_root_module,
                                                  jl_symbol("Core"));
     jl_base_module = (jl_module_t*)jl_get_global(jl_root_module,
                                                  jl_symbol("Base"));
-    jl_main_module = (jl_module_t*)jl_get_global(jl_root_module,
-                                                 jl_symbol("Main"));
+    jl_current_module = jl_base_module;
 
     jl_array_t *idtl = (jl_array_t*)jl_deserialize_value(&f);
     // rehash ObjectIdDicts

src/julia.h

@@ -700,6 +700,7 @@ void jl_cell_1d_push(jl_array_t *a, jl_value_t *item);
 // system information
 DLLEXPORT int jl_errno(void);
 DLLEXPORT jl_value_t *jl_strerror(int errnum);
+DLLEXPORT int32_t jl_stat(const char* path, char* statbuf);
 
 // environment entries
 DLLEXPORT jl_value_t *jl_environ(int i);
@@ -795,7 +796,6 @@ void jl_compile(jl_function_t *f);
 void jl_generate_fptr(jl_function_t *f);
 DLLEXPORT jl_value_t *jl_toplevel_eval(jl_value_t *v);
 jl_value_t *jl_eval_global_var(jl_module_t *m, jl_sym_t *e);
-char *jl_find_file_in_path(const char *fname);
 DLLEXPORT void jl_load(const char *fname);
 void jl_parse_eval_all(char *fname);
 jl_value_t *jl_interpret_toplevel_thunk(jl_lambda_info_t *lam);

src/toplevel.c

@@ -42,16 +42,14 @@ jl_value_t *jl_eval_module_expr(jl_expr_t *ex, int *plineno)
     jl_binding_t *b = jl_get_binding_wr(parent_module, name);
     jl_declare_constant(b);
     if (b->value != NULL) {
-        JL_PRINTF(JL_STDERR, "Warning: redefinition of module %s ignored\n",
-                   name->name);
-        return jl_nothing;
+        JL_PRINTF(JL_STDERR, "Warning: replacing module %s\n", name->name);
     }
     jl_module_t *newm = jl_new_module(name);
     b->value = (jl_value_t*)newm;
-    if (parent_module == jl_root_module && name == jl_symbol("Base")) {
-        // pick up Base module during bootstrap, and stay within it
-        // after loading.
-        jl_base_module = last_module = newm;
+    if (parent_module == jl_root_module && name == jl_symbol("Base") &&
+        jl_base_module == NULL) {
+        // pick up Base module during bootstrap
+        jl_base_module = newm;
     }
     JL_GC_PUSH(&last_module);
     jl_current_module = newm;
@@ -77,6 +75,17 @@ jl_value_t *jl_eval_module_expr(jl_expr_t *ex, int *plineno)
     }
     JL_GC_POP();
     jl_current_module = last_module;
+
+    // remove non-exported macros
+    size_t i;
+    void **table = newm->bindings.table;
+    for(i=1; i < newm->bindings.size; i+=2) {
+        if (table[i] != HT_NOTFOUND) {
+            jl_binding_t *b = (jl_binding_t*)table[i];
+            if (b->name->name[0]=='@' && !b->exportp)
+                b->value = NULL;
+        }
+    }
     return jl_nothing;
 }
 
@@ -311,28 +320,13 @@ void jl_parse_eval_all(char *fname)
 
 int asprintf(char **strp, const char *fmt, ...);
 
-// fpath needs to be freed if != fname
-char *jl_find_file_in_path(const char *fname)
+void jl_load(const char *fname)
 {
     char *fpath = (char*)fname;
-    int fid = open (fpath, O_RDONLY);
-    // try adding julia home
-    if (fid == -1 && julia_home && fname[0] != '/') {
-        if (-1 != asprintf(&fpath, "%s/../lib/julia/%s", julia_home, fname))
-            fid = open (fpath, O_RDONLY);
+    struct stat stbuf;
+    if (jl_stat(fpath, (char*)&stbuf) != 0) {
+        jl_errorf("could not open file %s", fpath);
     }
-    if (fid == -1) {
-        if (fpath != fname) free(fpath);
-        jl_errorf("could not open file %s", fname);
-    }
-    close(fid);
-
-    return fpath;
-}
-
-void jl_load(const char *fname)
-{
-    char *fpath = jl_find_file_in_path(fname);
     jl_start_parsing_file(fpath);
     jl_parse_eval_all(fpath);
     if (fpath != fname) free(fpath);

ui/repl.c

@@ -98,17 +98,32 @@ void parse_opts(int *argcp, char ***argvp) {
             program = (*argvp)[0];
         }
     }
+    if (image_file) {
+        int build_time_path = 0;
 #ifdef JL_SYSTEM_IMAGE_PATH
-    if (image_file && !imagepathspecified) {
-        image_file = JL_SYSTEM_IMAGE_PATH;
+        if (!imagepathspecified) {
+            image_file = JL_SYSTEM_IMAGE_PATH;
+            build_time_path = 1;
+        }
+#endif
         if (image_file[0] != PATHSEP) {
+            struct stat stbuf;
             char path[512];
-            snprintf(path, sizeof(path), "%s%s%s",
-                     julia_home, PATHSEPSTRING, JL_SYSTEM_IMAGE_PATH);
-            image_file = strdup(path);
+            if (build_time_path) {
+                // build time path relative to JULIA_HOME
+                snprintf(path, sizeof(path), "%s%s%s",
+                         julia_home, PATHSEPSTRING, JL_SYSTEM_IMAGE_PATH);
+                image_file = strdup(path);
+            }
+            else if (jl_stat(image_file, (char*)&stbuf) != 0) {
+                // otherwise try julia_home/../lib/julia/%s
+                snprintf(path, sizeof(path), "%s%s..%slib%sjulia%s%s",
+                         julia_home, PATHSEPSTRING, PATHSEPSTRING,
+                         PATHSEPSTRING, PATHSEPSTRING, image_file);
+                image_file = strdup(path);
+            }
         }
     }
-#endif
 }
 
 int ends_with_semicolon(const char *input)