From 3e85612f4dfcb630e3e395f6c4bd6630ec8c259c Mon Sep 17 00:00:00 2001 From: Milan Bouchet-Valat Date: Sat, 4 May 2024 11:47:39 +0200 Subject: [PATCH 1/5] Move compiled cache from DATAROOTDIR to LIBDIR The FHS specifies that only architecture-independent files should be put under /usr/share (DATAROOTDIR). Originally we only stored .ji files in the "compiled" directory, but now we also put .so files. Keep /usr/share and /usr/local/share in the default `DEPOT_PATH` for backward compatibility and because they can still be used for architecture-independent files such as environments or registries. Also fix a few uses of `"data"` which should have been "`DATAROOTDIR`" and wouldn't work if using a different `DATAROOTDIR` when building. --- base/initdefs.jl | 28 ++++++++++++++++++++-------- base/sysinfo.jl | 2 +- contrib/generate_precompile.jl | 2 +- doc/make.jl | 2 +- doc/src/devdocs/EscapeAnalysis.md | 2 +- pkgimage.mk | 2 +- 6 files changed, 25 insertions(+), 13 deletions(-) diff --git a/base/initdefs.jl b/base/initdefs.jl index 182984ae01d56..21e13353d1a3b 100644 --- a/base/initdefs.jl +++ b/base/initdefs.jl @@ -50,12 +50,20 @@ environments, repo clones, cached compiled package images, and configuration files. By default it includes: 1. `~/.julia` where `~` is the user home as appropriate on the system; -2. an architecture-specific shared system directory, e.g. `/usr/local/share/julia`; -3. an architecture-independent shared system directory, e.g. `/usr/share/julia`. - -So `DEPOT_PATH` might be: +2. an architecture-specific shared system directory specific to the local host, + e.g. `/usr/local/lib/julia`; +3. an architecture-independent shared system directory specific to the local host, + e.g. `/usr/local/share/julia`; +4. an architecture-specific shared system directory, + e.g. `/usr/lib/julia`; +5. an architecture-independent shared system directory, + e.g. `/usr/share/julia` + +All directories except the first are relative to the path of the `julia` executable. +So if Julia is installed to `/usr/bin/julia`, `DEPOT_PATH` might be: ```julia -[joinpath(homedir(), ".julia"), "/usr/local/share/julia", "/usr/share/julia"] +[joinpath(homedir(), ".julia"), "/usr/local/lib/julia", "/usr/local/share/julia", + "/usr/lib/julia", "/usr/share/julia"] ``` The first entry is the "user depot" and should be writable by and owned by the current user. The user depot is where: registries are cloned, new package versions @@ -76,7 +84,7 @@ Here is an overview of some of the subdirectories that may exist in a depot: * `artifacts`: Contains content that packages use for which Pkg manages the installation of. * `clones`: Contains full clones of package repos. Maintained by `Pkg.jl` and used as a cache. * `config`: Contains julia-level configuration such as a `startup.jl`. -* `compiled`: Contains precompiled `*.ji` files for packages. Maintained by Julia. +* `compiled`: Contains precompiled `*.ji` and `*.so` files for packages. Maintained by Julia. * `dev`: Default directory for `Pkg.develop`. Maintained by `Pkg.jl` and the user. * `environments`: Default package environments. For instance the global environment for a specific julia version. Maintained by `Pkg.jl`. * `logs`: Contains logs of `Pkg` and `REPL` operations. Maintained by `Pkg.jl` and Julia. @@ -95,9 +103,13 @@ See also [`JULIA_DEPOT_PATH`](@ref JULIA_DEPOT_PATH), and const DEPOT_PATH = String[] function append_bundled_depot_path!(DEPOT_PATH) - path = abspath(Sys.BINDIR, "..", "local", "share", "julia") + path = abspath(Sys.BINDIR, "..", "local", "foo", LIBDIR, "julia") + path in DEPOT_PATH || push!(DEPOT_PATH, path) + path = abspath(Sys.BINDIR, "..", "local", "foo", DATAROOTDIR, "julia") + path in DEPOT_PATH || push!(DEPOT_PATH, path) + path = abspath(Sys.BINDIR, LIBDIR, "julia") path in DEPOT_PATH || push!(DEPOT_PATH, path) - path = abspath(Sys.BINDIR, "..", "share", "julia") + path = abspath(Sys.BINDIR, DATAROOTDIR, "julia") path in DEPOT_PATH || push!(DEPOT_PATH, path) return DEPOT_PATH end diff --git a/base/sysinfo.jl b/base/sysinfo.jl index 484ac4d01d104..c80db8772a61c 100644 --- a/base/sysinfo.jl +++ b/base/sysinfo.jl @@ -166,7 +166,7 @@ end function __init_build() global BINDIR = ccall(:jl_get_julia_bindir, Any, ())::String vers = "v$(VERSION.major).$(VERSION.minor)" - global STDLIB = abspath(BINDIR, "..", "share", "julia", "stdlib", vers) + global STDLIB = abspath(BINDIR, Base.DATAROOTDIR, "julia", "stdlib", vers) nothing end diff --git a/contrib/generate_precompile.jl b/contrib/generate_precompile.jl index 1152e20d16842..8462adf25033f 100644 --- a/contrib/generate_precompile.jl +++ b/contrib/generate_precompile.jl @@ -12,7 +12,7 @@ Sys.__init_build() if !isdefined(Base, :uv_eventloop) Base.reinit_stdio() end -Base.include(@__MODULE__, joinpath(Sys.BINDIR, "..", "share", "julia", "test", "testhelpers", "FakePTYs.jl")) +Base.include(@__MODULE__, joinpath(Sys.BINDIR, Base.DATAROOTDIR, "julia", "test", "testhelpers", "FakePTYs.jl")) import .FakePTYs: open_fake_pty using Base.Meta diff --git a/doc/make.jl b/doc/make.jl index 15205d6c2f89d..1e98843a3e47c 100644 --- a/doc/make.jl +++ b/doc/make.jl @@ -4,7 +4,7 @@ empty!(LOAD_PATH) push!(LOAD_PATH, @__DIR__, "@stdlib") empty!(DEPOT_PATH) push!(DEPOT_PATH, joinpath(@__DIR__, "deps")) -push!(DEPOT_PATH, abspath(Sys.BINDIR, "..", "share", "julia")) +push!(DEPOT_PATH, abspath(Sys.BINDIR, Base.LIBDIR, "julia")) using Pkg Pkg.instantiate() diff --git a/doc/src/devdocs/EscapeAnalysis.md b/doc/src/devdocs/EscapeAnalysis.md index 1bd7868790f7f..14da1ecbcb6e3 100644 --- a/doc/src/devdocs/EscapeAnalysis.md +++ b/doc/src/devdocs/EscapeAnalysis.md @@ -20,7 +20,7 @@ This escape analysis aims to: You can give a try to the escape analysis by loading the `EAUtils.jl` utility script that defines the convenience entries `code_escapes` and `@code_escapes` for testing and debugging purposes: ```@repl EAUtils -let JULIA_DIR = normpath(Sys.BINDIR, "..", "share", "julia") +let JULIA_DIR = normpath(Sys.BINDIR, Base.DATAROOTDIR, "julia") # load `EscapeAnalysis` module to define the core analysis code include(normpath(JULIA_DIR, "base", "compiler", "ssair", "EscapeAnalysis", "EscapeAnalysis.jl")) using .EscapeAnalysis diff --git a/pkgimage.mk b/pkgimage.mk index 740b9760cab48..9fe4c72c646a8 100644 --- a/pkgimage.mk +++ b/pkgimage.mk @@ -6,7 +6,7 @@ include $(JULIAHOME)/stdlib/stdlib.mk # set some influential environment variables -export JULIA_DEPOT_PATH := $(shell echo $(call cygpath_w,$(build_prefix)/share/julia)) +export JULIA_DEPOT_PATH := $(shell echo $(call cygpath_w,$(build_libdir)/julia)) export JULIA_LOAD_PATH := @stdlib$(PATHSEP)$(shell echo $(call cygpath_w,$(JULIAHOME)/stdlib)) unexport JULIA_PROJECT := unexport JULIA_BINDIR := From 1def6ec91ee4c84f6e96c3128d9dcc88467cdf14 Mon Sep 17 00:00:00 2001 From: Milan Bouchet-Valat Date: Thu, 6 Nov 2025 22:53:12 +0100 Subject: [PATCH 2/5] Tentative fix --- test/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/Makefile b/test/Makefile index 61946e650dee5..f6199fdc12a44 100644 --- a/test/Makefile +++ b/test/Makefile @@ -6,7 +6,7 @@ VERSDIR := v$(shell cut -d. -f1-2 < $(JULIAHOME)/VERSION) STDLIBDIR := $(build_datarootdir)/julia/stdlib/$(VERSDIR) # TODO: this Makefile ignores BUILDDIR, except for computing JULIA_EXECUTABLE -export JULIA_DEPOT_PATH := $(build_prefix)/share/julia +export JULIA_DEPOT_PATH := $(shell echo $(call cygpath_w,$(build_private_libdir))) export JULIA_LOAD_PATH := @$(PATHSEP)@stdlib unexport JULIA_PROJECT := unexport JULIA_BINDIR := From 1a3311caf09eb8ea310ee4eb12dd97d67f59c4af Mon Sep 17 00:00:00 2001 From: Milan Bouchet-Valat Date: Fri, 7 Nov 2025 19:55:16 +0100 Subject: [PATCH 3/5] WIP --- Makefile | 2 ++ base/methodshow.jl | 2 +- base/sysinfo.jl | 2 +- test/Makefile | 4 ++-- 4 files changed, 6 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index 462f7dddd161b..0ecd0f2ac0090 100644 --- a/Makefile +++ b/Makefile @@ -417,6 +417,8 @@ ifeq ($(JULIA_BUILD_MODE),release) else ifeq ($(JULIA_BUILD_MODE),debug) $(INSTALL_M) $(build_private_libdir)/sys-debug.$(SHLIB_EXT) $(DESTDIR)$(private_libdir) endif + # Copy depot + cp -R -L $(build_private_libdir)/compiled/ $(DESTDIR)$(private_libdir) # Copy in all .jl sources as well mkdir -p $(DESTDIR)$(datarootdir)/julia/base $(DESTDIR)$(datarootdir)/julia/test diff --git a/base/methodshow.jl b/base/methodshow.jl index 075fcb2e891b6..1cf9195ddfebb 100644 --- a/base/methodshow.jl +++ b/base/methodshow.jl @@ -140,7 +140,7 @@ function fixup_stdlib_path(path::String) if isdefined(@__MODULE__, :Core) && isdefined(Core, :Compiler) compiler_folder = dirname(String(Base.moduleloc(Core.Compiler).file)) if dirname(path) == compiler_folder - return abspath(Sys.STDLIB, "..", "..", "Compiler", "src", basename(path)) + return abspath(Sys.BINDIR, Base.DATAROOTDIR, "julia", "Compiler", "src", basename(path)) end end end diff --git a/base/sysinfo.jl b/base/sysinfo.jl index f0190db0a3f41..d7b161a5e788e 100644 --- a/base/sysinfo.jl +++ b/base/sysinfo.jl @@ -55,7 +55,7 @@ global BINDIR::String = ccall(:jl_get_julia_bindir, Any, ())::String A string containing the full path to the directory containing the `stdlib` packages. """ -global STDLIB::String = "$BINDIR/../share/julia/stdlib/v$(VERSION.major).$(VERSION.minor)" # for bootstrap +global STDLIB::String = "$BINDIR/$DATAROOTDIR/julia/stdlib/v$(VERSION.major).$(VERSION.minor)" # for bootstrap # In case STDLIB change after julia is built, the variable below can be used # to update cached method locations to updated ones. const BUILD_STDLIB_PATH = STDLIB diff --git a/test/Makefile b/test/Makefile index f6199fdc12a44..6219417bbb724 100644 --- a/test/Makefile +++ b/test/Makefile @@ -46,7 +46,7 @@ relocatedepot: @cd $(SRCDIR) && \ $(call PRINT_JULIA, $(call spawn,$(JULIA_EXECUTABLE)) $(TEST_JULIA_OPTIONS) ./runtests.jl $(TEST_SCRIPT_OPTIONS) $@) @mkdir $(SRCDIR)/relocatedepot - @cp -R $(build_datarootdir)/julia $(SRCDIR)/relocatedepot + @cp -R $(build_private_libdir) $(SRCDIR)/relocatedepot @cp -R $(SRCDIR)/RelocationTestPkg1 $(SRCDIR)/relocatedepot @cp -R $(SRCDIR)/RelocationTestPkg2 $(SRCDIR)/relocatedepot @cp -R $(SRCDIR)/RelocationTestPkg3 $(SRCDIR)/relocatedepot @@ -59,7 +59,7 @@ revise-relocatedepot: revise-% : @cd $(SRCDIR) && \ $(call PRINT_JULIA, $(call spawn,$(JULIA_EXECUTABLE)) $(TEST_JULIA_OPTIONS) ./runtests.jl $(TEST_SCRIPT_OPTIONS) --revise $*) @mkdir $(SRCDIR)/relocatedepot - @cp -R $(build_datarootdir)/julia $(SRCDIR)/relocatedepot + @cp -R $(build_private_libdir) $(SRCDIR)/relocatedepot @cp -R $(SRCDIR)/RelocationTestPkg1 $(SRCDIR)/relocatedepot @cp -R $(SRCDIR)/RelocationTestPkg2 $(SRCDIR)/relocatedepot @cp -R $(SRCDIR)/RelocationTestPkg3 $(SRCDIR)/relocatedepot From 5cdd0176ef63b219afb3e4c30a08b926e4fc67a3 Mon Sep 17 00:00:00 2001 From: Milan Bouchet-Valat Date: Mon, 10 Nov 2025 23:27:06 +0100 Subject: [PATCH 4/5] Fix relocatedepot test --- pkgimage.mk | 2 +- test/Makefile | 3 ++- test/relocatedepot.jl | 14 +++++--------- 3 files changed, 8 insertions(+), 11 deletions(-) diff --git a/pkgimage.mk b/pkgimage.mk index 27bedac65d454..63a4db9722046 100644 --- a/pkgimage.mk +++ b/pkgimage.mk @@ -5,7 +5,7 @@ include $(JULIAHOME)/Make.inc include $(JULIAHOME)/stdlib/stdlib.mk # set some influential environment variables -export JULIA_DEPOT_PATH := $(shell echo $(call cygpath_w,$(build_private_libdir))) +export JULIA_DEPOT_PATH := $(shell echo $(call cygpath_w,$(build_private_libdir)))$(PATHSEP)$(shell echo $(call cygpath_w,$(build_datarootdir)/julia)) export JULIA_LOAD_PATH := @stdlib$(PATHSEP)$(shell echo $(call cygpath_w,$(JULIAHOME)/stdlib)) unexport JULIA_PROJECT := unexport JULIA_BINDIR := diff --git a/test/Makefile b/test/Makefile index 6219417bbb724..d0fa7adabed9b 100644 --- a/test/Makefile +++ b/test/Makefile @@ -46,7 +46,8 @@ relocatedepot: @cd $(SRCDIR) && \ $(call PRINT_JULIA, $(call spawn,$(JULIA_EXECUTABLE)) $(TEST_JULIA_OPTIONS) ./runtests.jl $(TEST_SCRIPT_OPTIONS) $@) @mkdir $(SRCDIR)/relocatedepot - @cp -R $(build_private_libdir) $(SRCDIR)/relocatedepot + @cp -R $(build_datarootdir)/julia/* $(SRCDIR)/relocatedepot + @cp -R $(build_private_libdir)/compiled $(SRCDIR)/relocatedepot @cp -R $(SRCDIR)/RelocationTestPkg1 $(SRCDIR)/relocatedepot @cp -R $(SRCDIR)/RelocationTestPkg2 $(SRCDIR)/relocatedepot @cp -R $(SRCDIR)/RelocationTestPkg3 $(SRCDIR)/relocatedepot diff --git a/test/relocatedepot.jl b/test/relocatedepot.jl index e8758365e3ff4..4ef9833d08549 100644 --- a/test/relocatedepot.jl +++ b/test/relocatedepot.jl @@ -252,7 +252,7 @@ else @testset "load stdlib from test/relocatedepot" begin test_harness() do push!(LOAD_PATH, "@stdlib") - push!(DEPOT_PATH, joinpath(@__DIR__, "relocatedepot", "julia")) + push!(DEPOT_PATH, joinpath(@__DIR__, "relocatedepot")) # stdlib should be already precompiled pkg = Base.identify_package("DelimitedFiles") @test Base.isprecompiled(pkg) == true @@ -264,8 +264,7 @@ else pkgname = "RelocationTestPkg1" test_harness() do push!(LOAD_PATH, joinpath(@__DIR__, "relocatedepot")) - push!(DEPOT_PATH, joinpath(@__DIR__, "relocatedepot")) # required to find src files - push!(DEPOT_PATH, joinpath(@__DIR__, "relocatedepot", "julia")) # contains cache file + push!(DEPOT_PATH, joinpath(@__DIR__, "relocatedepot")) pkg = Base.identify_package(pkgname) @test Base.isprecompiled(pkg) == true @test Base.isrelocatable(pkg) == true @@ -276,8 +275,7 @@ else pkgname = "RelocationTestPkg2" test_harness() do push!(LOAD_PATH, joinpath(@__DIR__, "relocatedepot")) - push!(DEPOT_PATH, joinpath(@__DIR__, "relocatedepot")) # required to find src files - push!(DEPOT_PATH, joinpath(@__DIR__, "relocatedepot", "julia")) # contains cache file + push!(DEPOT_PATH, joinpath(@__DIR__, "relocatedepot")) pkg = Base.identify_package(pkgname) @test Base.isprecompiled(pkg) == false # moving depot changes mtime of include_dependency @test Base.isrelocatable(pkg) == false # because not precompiled @@ -294,8 +292,7 @@ else pkgname = "RelocationTestPkg3" test_harness() do push!(LOAD_PATH, joinpath(@__DIR__, "relocatedepot")) - push!(DEPOT_PATH, joinpath(@__DIR__, "relocatedepot")) # required to find src files - push!(DEPOT_PATH, joinpath(@__DIR__, "relocatedepot", "julia")) # contains cache file + push!(DEPOT_PATH, joinpath(@__DIR__, "relocatedepot")) pkg = Base.identify_package(pkgname) @test Base.isprecompiled(pkg) == true @test Base.isrelocatable(pkg) == true @@ -309,8 +306,7 @@ else pkgname = "RelocationTestPkg4" test_harness() do push!(LOAD_PATH, @__DIR__, "relocatedepot") - push!(DEPOT_PATH, joinpath(@__DIR__, "relocatedepot")) # required to find src files - push!(DEPOT_PATH, joinpath(@__DIR__, "relocatedepot", "julia")) # contains cache file + push!(DEPOT_PATH, joinpath(@__DIR__, "relocatedepot")) pkg = Base.identify_package(pkgname) # precompiled but not relocatable @test Base.isprecompiled(pkg) == true From 879f1df850b67343dbef6d40fd242a5bbbf5fb1e Mon Sep 17 00:00:00 2001 From: Milan Bouchet-Valat Date: Mon, 10 Nov 2025 23:31:13 +0100 Subject: [PATCH 5/5] Fix --- test/Makefile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/Makefile b/test/Makefile index d0fa7adabed9b..8b241255c36e7 100644 --- a/test/Makefile +++ b/test/Makefile @@ -60,7 +60,8 @@ revise-relocatedepot: revise-% : @cd $(SRCDIR) && \ $(call PRINT_JULIA, $(call spawn,$(JULIA_EXECUTABLE)) $(TEST_JULIA_OPTIONS) ./runtests.jl $(TEST_SCRIPT_OPTIONS) --revise $*) @mkdir $(SRCDIR)/relocatedepot - @cp -R $(build_private_libdir) $(SRCDIR)/relocatedepot + @cp -R $(build_datarootdir)/julia/* $(SRCDIR)/relocatedepot + @cp -R $(build_private_libdir)/compiled $(SRCDIR)/relocatedepot @cp -R $(SRCDIR)/RelocationTestPkg1 $(SRCDIR)/relocatedepot @cp -R $(SRCDIR)/RelocationTestPkg2 $(SRCDIR)/relocatedepot @cp -R $(SRCDIR)/RelocationTestPkg3 $(SRCDIR)/relocatedepot