Skip to content

loading: Delete normpath where possible #60251

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 1 commit into
base: master
Choose a base branch
from
Open

loading: Delete normpath where possible #60251

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 4 additions & 4 deletions base/initdefs.jl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -302,15 +302,15 @@ function load_path_expand(env::AbstractString)::Union{String, Nothing}
path = joinpath(depot, "environments", name)
isdir(path) || continue
for proj in project_names
file = abspath(path, proj)
file = joinpath(path, proj)
isfile_casesensitive(file) && return file
end
end
isempty(DEPOT_PATH) && return nothing
return abspath(DEPOT_PATH[1], "environments", name, project_names[end])
return abspath(DEPOT_PATH[1], "environments", name, project_names[end]; safe=true)
end
# otherwise, it's a path
path = abspath(env)
path = abspath(env; safe=true)
if isdir(path)
# directory with a project file?
for proj in project_names
Expand All @@ -337,7 +337,7 @@ function active_project(search_load_path::Bool=true)
# there are backedges here from abspath(::AbstractString, ::String)
project = project::String
if !isfile_casesensitive(project) && basename(project) ∉ project_names
project = abspath(project, "Project.toml")
project = abspath(project, "Project.toml"; safe=true)
end
return project
end
Expand Down
78 changes: 51 additions & 27 deletions base/loading.jl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -964,7 +964,7 @@ function project_file_manifest_path(project_file::String)::Union{Nothing,String}
manifest_path = get(cache.project_file_manifest_path, project_file, missing)
manifest_path === missing || return manifest_path
end
dir = abspath(dirname(project_file))
dir = abspath(dirname(project_file); safe=true)
isfile_casesensitive(project_file) || return nothing
d = parsed_toml(project_file)
base_manifest = workspace_manifest(project_file)
Expand All @@ -974,7 +974,7 @@ function project_file_manifest_path(project_file::String)::Union{Nothing,String}
explicit_manifest = get(d, "manifest", nothing)::Union{String, Nothing}
manifest_path = nothing
if explicit_manifest !== nothing
manifest_file = normpath(joinpath(dir, explicit_manifest))
manifest_file = joinpath(dir, explicit_manifest)
if isfile_casesensitive(manifest_file)
manifest_path = manifest_file
end
Expand All @@ -998,10 +998,10 @@ end
# given a directory (implicit env from LOAD_PATH) and a name,
# check if it is an implicit package
function entry_point_and_project_file_inside(dir::String, name::String)::Union{Tuple{Nothing,Nothing},Tuple{String,Nothing},Tuple{String,String}}
path = normpath(joinpath(dir, "src", "$name.jl"))
path = joinpath(dir, "src", "$name.jl")
isfile_casesensitive(path) || return nothing, nothing
for proj in project_names
project_file = normpath(joinpath(dir, proj))
project_file = joinpath(dir, proj)
isfile_casesensitive(project_file) || continue
return path, project_file
end
Expand All @@ -1018,7 +1018,7 @@ function entry_point_and_project_file(dir::String, name::String)::Union{Tuple{No
path, project_file = entry_point_and_project_file_inside(dir_jl, name)
path === nothing || return path, project_file
# check for less likely case with a bare file and no src directory last to minimize stat calls
path = normpath(joinpath(dir, "$name.jl"))
path = joinpath(dir, "$name.jl")
isfile_casesensitive(path) && return path, nothing
return nothing, nothing
end
Expand All @@ -1038,9 +1038,9 @@ end

# given a path, name, and possibly an entryfile, return the entry point
function entry_path(path::String, name::String, entryfile::Union{Nothing,String})::String
isfile_casesensitive(path) && return normpath(path)
isfile_casesensitive(path) && return path
entrypoint = entryfile === nothing ? joinpath("src", "$name.jl") : entryfile
return normpath(joinpath(path, entrypoint))
return joinpath(path, entrypoint)
end

## explicit project & manifest API ##
Expand Down Expand Up @@ -1204,7 +1204,7 @@ function explicit_manifest_uuid_load_spec(project_file::String, pkg::PkgId)::Uni
error("failed to find source of parent package: \"$name\"")
end
parent_path = parent_load_spec.path
p = normpath(dirname(parent_path), "..")
p = joinpath(dirname(parent_path), "..")
return PkgLoadSpec(find_ext_path(p, pkg.name), parent_load_spec.julia_syntax_version)
end
end
Expand All @@ -1228,7 +1228,7 @@ function explicit_manifest_entry_load_spec(manifest_file::String, pkg::PkgId, en
path = get(entry, "path", nothing)::Union{Nothing, String}
entryfile = get(entry, "entryfile", nothing)::Union{Nothing, String}
if path !== nothing
path = entry_path(normpath(abspath(dirname(manifest_file), path)), pkg.name, entryfile)
path = entry_path(joinpath(dirname(manifest_file), path), pkg.name, entryfile)
return PkgLoadSpec(path, syntax_version)
end
hash = get(entry, "git-tree-sha1", nothing)::Union{Nothing, String}
Expand All @@ -1248,7 +1248,7 @@ function explicit_manifest_entry_load_spec(manifest_file::String, pkg::PkgId, en
for slug in (version_slug(uuid, hash), version_slug(uuid, hash, 4))
for depot in DEPOT_PATH
path = joinpath(depot, "packages", pkg.name, slug)
ispath(path) && return PkgLoadSpec(entry_path(abspath(path), pkg.name, entryfile), syntax_version)
ispath(path) && return PkgLoadSpec(entry_path(abspath(path; safe=true), pkg.name, entryfile), syntax_version)
end
end
# no depot contains the package, return missing to stop looking
Expand Down Expand Up @@ -2394,9 +2394,9 @@ function _include_dependency!(dep_list::Vector{Any}, track_dependencies::Bool,
track_content::Bool, path_may_be_dir::Bool)
prev = source_path(nothing)
if prev === nothing
path = abspath(_path)
path = abspath(_path; safe=true)
else
path = normpath(joinpath(dirname(prev), _path))
path = normpath(joinpath(dirname(prev), _path); safe=true)
end
if !track_dependencies[]
if !path_may_be_dir && !isfile(path)
Expand Down Expand Up @@ -3003,9 +3003,9 @@ function require_stdlib(package_uuidkey::PkgId, ext::Union{Nothing, String}, fro
if from_stdlib
# first since this is a stdlib, try to look there directly first
if ext === nothing
sourcepath = normpath(env, this_uuidkey.name, "src", this_uuidkey.name * ".jl")
sourcepath = joinpath(env, this_uuidkey.name, "src", this_uuidkey.name * ".jl")
else
sourcepath = find_ext_path(normpath(joinpath(env, package_uuidkey.name)), ext)
sourcepath = find_ext_path(joinpath(env, package_uuidkey.name), ext)
end
set_pkgorigin_version_path(this_uuidkey, sourcepath)
newm = _require_search_from_serialized(this_uuidkey, PkgLoadSpec(sourcepath, VERSION), UInt128(0), false; DEPOT_PATH=depot_path)
Expand Down Expand Up @@ -3191,13 +3191,14 @@ function evalfile(path::AbstractString, args::Vector{String}=String[])
end
evalfile(path::AbstractString, args::Vector) = evalfile(path, String[args...])

# Used in Pkg.jl
function load_path_setup_code(load_path::Bool=true)
code = """
append!(empty!(Base.DEPOT_PATH), $(repr(map(abspath, DEPOT_PATH))))
append!(empty!(Base.DL_LOAD_PATH), $(repr(map(abspath, DL_LOAD_PATH))))
append!(empty!(Base.DEPOT_PATH), $(repr(map(x -> abspath(x; safe=true), DEPOT_PATH))))
append!(empty!(Base.DL_LOAD_PATH), $(repr(map(x -> abspath(x; safe=true), DL_LOAD_PATH))))
"""
if load_path
load_path = map(abspath, Base.load_path())
load_path = map(x -> abspath(x; safe=true), Base.load_path())
path_sep = Sys.iswindows() ? ';' : ':'
any(path -> path_sep in path, load_path) &&
error("LOAD_PATH entries cannot contain $(repr(path_sep))")
Expand Down Expand Up @@ -3283,9 +3284,9 @@ function create_expr_cache(pkg::PkgId, input::PkgLoadSpec, output::String, outpu
@nospecialize internal_stderr internal_stdout
rm(output, force=true) # Remove file if it exists
output_o === nothing || rm(output_o, force=true)
depot_path = String[abspath(x) for x in DEPOT_PATH]
dl_load_path = String[abspath(x) for x in DL_LOAD_PATH]
load_path = String[abspath(x) for x in Base.load_path()]
depot_path = String[abspath(x; safe=true) for x in DEPOT_PATH]
dl_load_path = String[abspath(x; safe=true) for x in DL_LOAD_PATH]
load_path = String[abspath(x; safe=true) for x in Base.load_path()]
# if pkg is a stdlib, append its parent Project.toml to the load path
triggers = get(EXT_PRIMED, pkg, nothing)
if triggers !== nothing
Expand Down Expand Up @@ -3343,7 +3344,7 @@ function create_expr_cache(pkg::PkgId, input::PkgLoadSpec, output::String, outpu
Base.track_nested_precomp($(_pkg_str(vcat(Base.precompilation_stack, pkg))))
Base.loadable_extensions = $(_pkg_str(loadable_exts))
Base.precompiling_extension = $(loading_extension)
Base.include_package_for_output($(_pkg_str(pkg)), $(repr(abspath(input.path))), $(repr(input.julia_syntax_version)), $(repr(depot_path)), $(repr(dl_load_path)),
Base.include_package_for_output($(_pkg_str(pkg)), $(repr(abspath(input.path; safe=true))), $(repr(input.julia_syntax_version)), $(repr(depot_path)), $(repr(dl_load_path)),
$(repr(load_path)), $(_pkg_str(concrete_deps)), $(repr(source_path(nothing))))
""")
close(io.in)
Expand Down Expand Up @@ -3372,7 +3373,7 @@ function compilecache_path(pkg::PkgId, prefs_hash::UInt64; flags::CacheFlags=Cac
cachepath = joinpath(DEPOT_PATH[1], entrypath)
isdir(cachepath) || mkpath(cachepath)
if pkg.uuid === nothing
abspath(cachepath, entryfile) * ".ji"
joinpath(cachepath, entryfile) * ".ji"
else
crc = _crc32c(project)
crc = _crc32c(unsafe_string(JLOptions().image_file), crc)
Expand All @@ -3387,7 +3388,7 @@ function compilecache_path(pkg::PkgId, prefs_hash::UInt64; flags::CacheFlags=Cac

crc = _crc32c(prefs_hash, crc)
project_precompile_slug = slug(crc, 5)
abspath(cachepath, string(entryfile, "_", project_precompile_slug, ".ji"))
joinpath(cachepath, string(entryfile, "_", project_precompile_slug, ".ji"))
end
end

Expand Down Expand Up @@ -3594,9 +3595,26 @@ function CacheHeaderIncludes(dep_tuple::Tuple{Module, String, UInt64, UInt32, Fl
return CacheHeaderIncludes(PkgId(dep_tuple[1]), dep_tuple[2:end]..., String[])
end

function replace_depot_path(path::AbstractString, depots::Vector{String}=normalize_depots_for_relocation())
function replace_depot_path(path::AbstractString, depots::Vector{Union{String, Tuple{String, String}}}=normalize_depots_for_relocation())
# We must handle several cases:
# 1. The depot itself is in a symlink'ed path
# 2. The files were symlinked into the depot
# 3. A combination of both
this_realpath = Filesystem.realpath_default(path, nothing)
realpath_differs = this_realpath !== path
for depot in depots
if startswith(path, string(depot, Filesystem.pathsep())) || path == depot
if isa(depot, Tuple{String, String})
depotrealpath = depot[2]
depot = depot[1]
else
depotrealpath = depot
end
if this_realpath !== nothing && (startswith(this_realpath, string(depotrealpath, Filesystem.pathsep())) || this_realpath == depotrealpath)
# Simplest case: The file's realpath is within the depot's realpath, simply replace and go on
path = replace(this_realpath, depotrealpath => "@depot"; count=1)
break
elseif realpath_differs && startswith(path, string(depot, Filesystem.pathsep())) || path == depot
# The file's original path was loaded from symlinks within the depot - n.b.: the path may still have `..` components
path = replace(path, depot => "@depot"; count=1)
break
end
Expand All @@ -3605,14 +3623,20 @@ function replace_depot_path(path::AbstractString, depots::Vector{String}=normali
end

function normalize_depots_for_relocation()
depots = String[]
depots = Union{String, Tuple{String, String}}[]
sizehint!(depots, length(DEPOT_PATH))
for d in DEPOT_PATH
isdir(d) || continue
if isdirpath(d)
d = dirname(d)
end
push!(depots, abspath(d))
depotrealpath = realpath(d)
depotabspath = abspath(d)
if depotrealpath != depotabspath
push!(depots, (depotabspath, depotrealpath))
else
push!(depots, depotrealpath)
end
end
return depots
end
Expand Down
86 changes: 68 additions & 18 deletions base/path.jl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -373,38 +373,55 @@ julia> joinpath(["/home/myuser", "example.jl"])
joinpath

"""
normpath(path::AbstractString)::String
normpath(path::AbstractString; safe=false)::String

Normalize a path, removing "." and ".." entries and changing "/" to the canonical path separator
for the system.
Normalize a path, removing "." and changing "/" to the canonical path separator for the system.
If `safe` is false, also removes ".." entries.

!!! warning
Normalization with `safe==false` may change the meaning of a path if the
directory immediately preceding a ".." entry is a symbolic link.
For example, if `a` is a symbolic link in `a/../b`, then this path will
resolve to `b` in the link target's parent directory. However,
`normpath("a/../b")` will return `b`, which will resolve `b` in the
current working directory.

!!! compat "Julia 1.14"
The `safe` keyword argument was added in Julia 1.14.

# Examples
```jldoctest
julia> normpath("/home/myuser/../example.jl")
"/home/example.jl"

julia> normpath("/home/myuser/../example.jl"; safe=true)
"/home/myuser/../example.jl"

julia> normpath("Documents/Julia") == joinpath("Documents", "Julia")
true
```
"""
function normpath(path::String)
function normpath(path::String; safe=false)
isabs = isabspath(path)
isdir = isdirpath(path)
drive, path = splitdrive(path)
parts = split(path, path_separator_re; keepempty=false)
filter!(!=("."), parts)
while true
clean = true
for j = 1:length(parts)-1
if parts[j] != ".." && parts[j+1] == ".."
deleteat!(parts, j:j+1)
clean = false
break
if !safe
while true
clean = true
for j = 1:length(parts)-1
if parts[j] != ".." && parts[j+1] == ".."
deleteat!(parts, j:j+1)
clean = false
break
end
end
clean && break
end
clean && break
end
if isabs
# N.B.: `..` at the beginning of a path may always be removed
while !isempty(parts) && parts[1] == ".."
popfirst!(parts)
end
Expand All @@ -427,13 +444,16 @@ end
Convert a set of paths to a normalized path by joining them together and removing
"." and ".." entries. Equivalent to `normpath(joinpath(path, paths...))`.
"""
normpath(a::AbstractString, b::AbstractString...) = normpath(joinpath(a,b...))
normpath(a::AbstractString, b::AbstractString...; kwargs...) = normpath(joinpath(a,b...); kwargs...)

"""
abspath(path::AbstractString)::String
abspath(path::AbstractString; safe=false)::String

Convert a path to an absolute path by adding the current directory if necessary.
Also normalizes the path as in [`normpath`](@ref).

The resulting path is normalized as in normpath. If `safe` is false, `..` entries
are also removed, but the resulting path may not refer to the same location. See
[`normpath`](@ref) for details.

# Examples

Expand All @@ -444,8 +464,11 @@ If you are in a directory called `JuliaExample` and the data you are using is tw
Which gives a path like `"/home/JuliaUser/data/"`.

See also [`joinpath`](@ref), [`pwd`](@ref), [`expanduser`](@ref).

!!! compat "Julia 1.14"
The `safe` keyword argument was added in Julia 1.14.
"""
function abspath(a::String)::String
function abspath(a::String; safe=false)::String
if !isabspath(a)
cwd = pwd()
a_drive, a_nodrive = splitdrive(a)
Expand All @@ -456,7 +479,7 @@ function abspath(a::String)::String
a = joinpath(cwd, a)
end
end
return normpath(a)
return normpath(a; safe)
end

"""
Expand All @@ -465,7 +488,7 @@ end
Convert a set of paths to an absolute path by joining them together and adding the
current directory if necessary. Equivalent to `abspath(joinpath(path, paths...))`.
"""
abspath(a::AbstractString, b::AbstractString...) = abspath(joinpath(a,b...))
abspath(a::AbstractString, b::AbstractString...; kwargs...) = abspath(joinpath(a,b...); kwargs...)

if Sys.iswindows()

Expand Down Expand Up @@ -513,6 +536,33 @@ function realpath(path::AbstractString)
end
end

"""
realpath_default(path::AbstractString, default)::String

Like `realpath(path)`, expect returns `default` instead of throwing an exception
if `path` does not exist in the filesystem.
"""
function realpath_default(path::AbstractString, @nospecialize(default))
req = Libc.malloc(_sizeof_uv_fs)
try
ret = ccall(:uv_fs_realpath, Cint,
(Ptr{Cvoid}, Ptr{Cvoid}, Cstring, Ptr{Cvoid}),
C_NULL, req, path, C_NULL)
if ret < 0
uv_fs_req_cleanup(req)
if ret == Base.UV_ENOENT
return default
end
uv_error("realpath_default($(repr(path)))", ret)
end
path = unsafe_string(ccall(:jl_uv_fs_t_ptr, Cstring, (Ptr{Cvoid},), req))
uv_fs_req_cleanup(req)
return path
finally
Libc.free(req)
end
end

if Sys.iswindows()
# on windows, ~ means "temporary file"
expanduser(path::AbstractString) = path
Expand Down
Loading