Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

RFC: break up monolithic sysimg build work #38119

Closed
wants to merge 1 commit into from
Closed

RFC: break up monolithic sysimg build work #38119

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
15 changes: 11 additions & 4 deletions Makefile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -149,9 +149,12 @@ $(build_datarootdir)/julia/julia-config.jl: $(JULIAHOME)/contrib/julia-config.jl
$(build_depsbindir)/stringreplace: $(JULIAHOME)/contrib/stringreplace.c | $(build_depsbindir)
@$(call PRINT_CC, $(HOSTCC) -o $(build_depsbindir)/stringreplace $(JULIAHOME)/contrib/stringreplace.c)

julia-base-cache: julia-sysimg-$(JULIA_BUILD_MODE) | $(DIRS) $(build_datarootdir)/julia
@JULIA_BINDIR=$(call cygpath_w,$(build_bindir)) $(call spawn, $(JULIA_EXECUTABLE) --startup-file=no $(call cygpath_w,$(JULIAHOME)/etc/write_base_cache.jl) \
$(call cygpath_w,$(build_datarootdir)/julia/base.cache))
julia-base-cache: $(build_datarootdir)/julia/base.cache

# this depends on the phony target julia-sysimg- to force rebuilding
$(build_datarootdir)/julia/base.cache: $(JULIAHOME)/etc/write_base_cache.jl julia-sysimg-$(JULIA_BUILD_MODE) | $(DIRS) $(build_datarootdir)/julia
@JULIA_BINDIR=$(call cygpath_w,$(build_bindir)) \
$(call spawn,$(JULIA_EXECUTABLE)) --startup-file=no $(call cygpath_w,$<) $(call cygpath_w,$@)

# public libraries, that are installed in $(prefix)/lib
JL_TARGETS := julia julialoader
Expand Down Expand Up @@ -359,8 +362,10 @@ endif
cp -R -L $(build_includedir)/julia/* $(DESTDIR)$(includedir)/julia
# Copy system image
$(INSTALL_M) $(build_private_libdir)/sys.$(SHLIB_EXT) $(DESTDIR)$(private_libdir)
$(INSTALL_M) $(build_private_libdir)/fullsys.$(SHLIB_EXT) $(DESTDIR)$(private_libdir)
ifeq ($(BUNDLE_DEBUG_LIBS),1)
$(INSTALL_M) $(build_private_libdir)/sys-debug.$(SHLIB_EXT) $(DESTDIR)$(private_libdir)
$(INSTALL_M) $(build_private_libdir)/fullsys-debug.$(SHLIB_EXT) $(DESTDIR)$(private_libdir)
endif

# Copy in all .jl sources as well
Expand Down Expand Up @@ -616,7 +621,9 @@ LLVM_SIZE := $(build_depsbindir)/llvm-size$(EXE)
endif
build-stats:
@printf $(JULCOLOR)' ==> ./julia binary sizes\n'$(ENDCOLOR)
$(call spawn,$(LLVM_SIZE) -A $(call cygpath_w,$(build_private_libdir)/sys.$(SHLIB_EXT)) \
$(call spawn,$(LLVM_SIZE) -A \
$(call cygpath_w,$(build_private_libdir)/fullsys.$(SHLIB_EXT)) \
$(call cygpath_w,$(build_private_libdir)/sys.$(SHLIB_EXT)) \
$(call cygpath_w,$(build_shlibdir)/libjulia.$(SHLIB_EXT)) \
$(call cygpath_w,$(build_bindir)/julia$(EXE)))
@printf $(JULCOLOR)' ==> ./julia launch speedtest\n'$(ENDCOLOR)
Expand Down
1 change: 0 additions & 1 deletion base/.gitignore
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,4 +7,3 @@
/uv_constants.jl
/version_git.jl
/version_git.jl.phony
/userimg.jl
7 changes: 7 additions & 0 deletions base/Base.jl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -424,4 +424,11 @@ end

end

# Ensure this file is also tracked
pushfirst!(_included_files, (Main, abspath(@__FILE__)))

end # baremodule Base

# Set up Main module
using .Base
import Base.MainInclude: eval, include
Copy link
Sponsor Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It'd be really nice if this can somehow be done elsewhere. The idea was to make Base more like a normal package; it should only create the Base module.

Copy link
Sponsor Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This can go elsewhere, but I thought it felt the most natural here (it means that if you load just -J Base.so, it acts just like you are just inside a fresh module Main, instead of being a baremodule Main).

6 changes: 3 additions & 3 deletions base/initdefs.jl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -94,6 +94,7 @@ function append_default_depot_path!(DEPOT_PATH)
path in DEPOT_PATH || push!(DEPOT_PATH, path)
path = abspath(Sys.BINDIR::String, "..", "share", "julia")
path in DEPOT_PATH || push!(DEPOT_PATH, path)
return DEPOT_PATH
end

function init_depot_path()
Expand All @@ -112,6 +113,7 @@ function init_depot_path()
else
append_default_depot_path!(DEPOT_PATH)
end
nothing
end

## LOAD_PATH & ACTIVE_PROJECT ##
Expand Down Expand Up @@ -217,9 +219,7 @@ function parse_load_path(str::String)
end

function init_load_path()
if Base.creating_sysimg
paths = ["@stdlib"]
elseif haskey(ENV, "JULIA_LOAD_PATH")
if haskey(ENV, "JULIA_LOAD_PATH")
paths = parse_load_path(ENV["JULIA_LOAD_PATH"])
else
paths = filter!(env -> env !== nothing,
Expand Down
131 changes: 0 additions & 131 deletions base/sysimg.jl
View file
Open in desktop

This file was deleted.

29 changes: 21 additions & 8 deletions doc/Manifest.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,10 +20,10 @@ uuid = "ffbed154-4ef7-542d-bbb7-c09d3a79fcae"
version = "0.8.3"

[[Documenter]]
deps = ["Base64", "Dates", "DocStringExtensions", "InteractiveUtils", "JSON", "LibGit2", "Logging", "Markdown", "REPL", "Test", "Unicode"]
git-tree-sha1 = "fb1ff838470573adc15c71ba79f8d31328f035da"
deps = ["Base64", "Dates", "DocStringExtensions", "IOCapture", "InteractiveUtils", "JSON", "LibGit2", "Logging", "Markdown", "REPL", "Test", "Unicode"]
git-tree-sha1 = "a4875e0763112d6d017126f3944f4133abb342ae"
uuid = "e30172f5-a6a5-5a46-863b-614d45cd2de4"
version = "0.25.2"
version = "0.25.5"

[[DocumenterLaTeX]]
deps = ["Documenter", "Test"]
Expand All @@ -32,9 +32,15 @@ uuid = "cd674d7a-5f81-5cf3-af33-235ef1834b99"
version = "0.2.0"

[[Downloads]]
deps = ["ArgTools", "LibCURL"]
deps = ["ArgTools", "LibCURL", "NetworkOptions"]
uuid = "f43a241f-c20a-4ad4-852c-f6b1247861c6"

[[IOCapture]]
deps = ["Logging"]
git-tree-sha1 = "377252859f740c217b936cebcd918a44f9b53b59"
uuid = "b5f81e59-6552-4d32-b1f0-c071b021bf89"
version = "0.1.1"

[[InteractiveUtils]]
deps = ["Markdown"]
uuid = "b77e0a4c-d291-57a0-90e8-8db25a27a240"
Expand All @@ -54,7 +60,7 @@ deps = ["Libdl"]
uuid = "deac9b47-8bc7-5906-a0fe-35ac56dc84c0"

[[LibGit2]]
deps = ["Printf"]
deps = ["NetworkOptions", "Printf"]
uuid = "76f85450-5226-5b5a-8eaa-529ad045b433"

[[Libdl]]
Expand All @@ -73,14 +79,17 @@ uuid = "a63ad114-7e13-5084-954f-fe012c677804"
[[MozillaCACerts_jll]]
uuid = "14a3606d-f60d-562e-9121-12d972cd8159"

[[NetworkOptions]]
uuid = "ca575930-c2e3-43a9-ace4-1e988b2c1908"

[[Parsers]]
deps = ["Dates"]
git-tree-sha1 = "6fa4202675c05ba0f8268a6ddf07606350eda3ce"
git-tree-sha1 = "b417be52e8be24e916e34b3d70ec2da7bdf56a68"
uuid = "69de0a69-1ddd-5017-9359-2bf0b02dc9f0"
version = "1.0.11"
version = "1.0.12"

[[Pkg]]
deps = ["Artifacts", "Dates", "Downloads", "LibGit2", "Libdl", "Logging", "Markdown", "Printf", "REPL", "Random", "SHA", "Serialization", "TOML", "UUIDs"]
deps = ["Artifacts", "Dates", "Downloads", "LibGit2", "Libdl", "Logging", "Markdown", "Printf", "REPL", "Random", "SHA", "Serialization", "TOML", "Tar", "UUIDs"]
uuid = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f"

[[Printf]]
Expand Down Expand Up @@ -108,6 +117,10 @@ uuid = "6462fe0b-24de-5631-8697-dd941f90decc"
deps = ["Dates"]
uuid = "fa267f1f-6049-4f14-aa54-33bafae1ed76"

[[Tar]]
deps = ["ArgTools", "SHA"]
uuid = "a4e569a6-e804-4fa4-b0f3-eef7a1d5b13e"

[[Test]]
deps = ["InteractiveUtils", "Logging", "Random", "Serialization"]
uuid = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
Expand Down
45 changes: 32 additions & 13 deletions src/init.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -540,7 +540,7 @@ static const char *absformat(const char *in)
return out;
}

static void jl_resolve_sysimg_location(JL_IMAGE_SEARCH rel)
static const char *jl_resolve_sysimg_location(JL_IMAGE_SEARCH rel, int just_repl)
{ // this function resolves the paths in jl_options to absolute file locations as needed
// and it replaces the pointers to `julia_bindir`, `julia_bin`, `image_file`, and output file paths
// it may fail, print an error, and exit(1) if any of these paths are longer than PATH_MAX
Expand Down Expand Up @@ -569,7 +569,24 @@ static void jl_resolve_sysimg_location(JL_IMAGE_SEARCH rel)
jl_options.julia_bindir = abspath(jl_options.julia_bindir, 0);
free(free_path);
free_path = NULL;
const char *image_file = NULL;
if (jl_options.image_file) {
if (jl_options.image_file == jl_get_default_sysimg_path() && just_repl) {
// if we're just launching a REPL for the default environment
// interpose a slightly more complete environment at startup
free_path = (char*)malloc_s(PATH_MAX);
int n = snprintf(free_path, PATH_MAX, "%s" PATHSEPSTRING "%sfull",
jl_options.julia_bindir, jl_options.image_file);
if (n >= PATH_MAX || n < 0) {
jl_error("fatal error: jl_options.image_file path too long");
}
// swap filename and "full" prefix in string to form the target name
char *dirp = strrchr(free_path, PATHSEPSTRING[0]) + strlen(PATHSEPSTRING);
memmove(dirp + 4, dirp, n - (dirp - free_path) - 4);
memcpy(dirp, "full", 4);
image_file = free_path;
free_path = NULL;
}
if (rel == JL_IMAGE_JULIA_HOME && !isabspath(jl_options.image_file)) {
// build time path, relative to JULIA_BINDIR
free_path = (char*)malloc_s(PATH_MAX);
Expand All @@ -579,13 +596,13 @@ static void jl_resolve_sysimg_location(JL_IMAGE_SEARCH rel)
jl_error("fatal error: jl_options.image_file path too long");
}
jl_options.image_file = free_path;
free_path = NULL;
}
if (jl_options.image_file)
else {
jl_options.image_file = abspath(jl_options.image_file, 0);
if (free_path) {
free(free_path);
free_path = NULL;
}
if (image_file == NULL)
image_file = jl_options.image_file;
}
if (jl_options.outputo)
jl_options.outputo = abspath(jl_options.outputo, 0);
Expand All @@ -609,6 +626,8 @@ static void jl_resolve_sysimg_location(JL_IMAGE_SEARCH rel)
}
}
}

return image_file;
}

static void jl_set_io_wait(int v)
Expand All @@ -626,7 +645,7 @@ static void restore_fp_env(void)
}
}

void _julia_init(JL_IMAGE_SEARCH rel)
void _julia_init(JL_IMAGE_SEARCH rel, int just_repl)
{
jl_init_timing();
// Make sure we finalize the tls callback before starting any threads.
Expand Down Expand Up @@ -728,15 +747,15 @@ void _julia_init(JL_IMAGE_SEARCH rel)

jl_gc_enable(0);

jl_resolve_sysimg_location(rel);
const char *image_file = jl_resolve_sysimg_location(rel, just_repl);
// loads sysimg if available, and conditionally sets jl_options.cpu_target
if (jl_options.image_file)
jl_preload_sysimg_so(jl_options.image_file);
if (image_file)
jl_preload_sysimg_so(image_file);
if (jl_options.cpu_target == NULL)
jl_options.cpu_target = "native";

if (jl_options.image_file) {
jl_restore_system_image(jl_options.image_file);
if (image_file) {
jl_restore_system_image(image_file);
}
else {
jl_init_types();
Expand All @@ -752,7 +771,7 @@ void _julia_init(JL_IMAGE_SEARCH rel)
jl_init_flisp();
jl_init_serializer();

if (!jl_options.image_file) {
if (!image_file) {
jl_core_module = jl_new_module(jl_symbol("Core"));
jl_core_module->parent = jl_core_module;
jl_type_typename->mt->module = jl_core_module;
Expand Down Expand Up @@ -786,7 +805,7 @@ void _julia_init(JL_IMAGE_SEARCH rel)

jl_gc_enable(1);

if (jl_options.image_file && (!jl_generating_output() || jl_options.incremental) && jl_module_init_order) {
if (image_file && (!jl_generating_output() || jl_options.incremental) && jl_module_init_order) {
jl_array_t *init_order = jl_module_init_order;
JL_GC_PUSH1(&init_order);
jl_module_init_order = NULL;
Expand Down