From 7dad444e7b73a5bc993ee3a3839c79acb8620fe4 Mon Sep 17 00:00:00 2001 From: DilumAluthgeBot <43731525+DilumAluthgeBot@users.noreply.github.com> Date: Fri, 16 Feb 2024 11:36:53 -0500 Subject: [PATCH 01/25] =?UTF-8?q?=F0=9F=A4=96=20[master]=20Bump=20the=20Sp?= =?UTF-8?q?arseArrays=20stdlib=20from=20c9f7293=20to=20cb602d7=20(#53361)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Stdlib: SparseArrays URL: https://github.com/JuliaSparse/SparseArrays.jl.git Stdlib branch: main Julia branch: master Old commit: c9f7293 New commit: cb602d7 Julia version: 1.12.0-DEV SparseArrays version: 1.11.0(Does not match) Bump invoked by: @dkarrasch Powered by: [BumpStdlibs.jl](https://github.com/JuliaLang/BumpStdlibs.jl) Diff: https://github.com/JuliaSparse/SparseArrays.jl/compare/c9f7293c10e6bea775feceabd4554ab55c34e3a8...cb602d7b7cf46057ddc87d23cda2bdd168a548ac ``` $ git log --oneline c9f7293..cb602d7 cb602d7 Add generic matmatmul for inplace sparse x sparse (#486) 95575c0 Speedup for `findmin()/findmax()` on sparse arrays (#510) (#511) 4cc31f2 Fix error message in getcommon() in cholmod (#509) 1748989 Move `fkeep!` docstring to the right function (#503) 1aa6431 Update ci.yml timeout (#507) 1f88ae1 Update ci.yml - add codecov token (#504) ``` Co-authored-by: Dilum Aluthge (cherry picked from commit ddd7afb95165b15f87b2c83e7862437d0a68272d) --- .../md5 | 1 - .../sha512 | 1 - .../md5 | 1 + .../sha512 | 1 + stdlib/SparseArrays.version | 2 +- 5 files changed, 3 insertions(+), 3 deletions(-) delete mode 100644 deps/checksums/SparseArrays-c9f7293c10e6bea775feceabd4554ab55c34e3a8.tar.gz/md5 delete mode 100644 deps/checksums/SparseArrays-c9f7293c10e6bea775feceabd4554ab55c34e3a8.tar.gz/sha512 create mode 100644 deps/checksums/SparseArrays-cb602d7b7cf46057ddc87d23cda2bdd168a548ac.tar.gz/md5 create mode 100644 deps/checksums/SparseArrays-cb602d7b7cf46057ddc87d23cda2bdd168a548ac.tar.gz/sha512 diff --git a/deps/checksums/SparseArrays-c9f7293c10e6bea775feceabd4554ab55c34e3a8.tar.gz/md5 b/deps/checksums/SparseArrays-c9f7293c10e6bea775feceabd4554ab55c34e3a8.tar.gz/md5 deleted file mode 100644 index 190ea5ea307d8..0000000000000 --- a/deps/checksums/SparseArrays-c9f7293c10e6bea775feceabd4554ab55c34e3a8.tar.gz/md5 +++ /dev/null @@ -1 +0,0 @@ -b765873c42674134d74758200885e454 diff --git a/deps/checksums/SparseArrays-c9f7293c10e6bea775feceabd4554ab55c34e3a8.tar.gz/sha512 b/deps/checksums/SparseArrays-c9f7293c10e6bea775feceabd4554ab55c34e3a8.tar.gz/sha512 deleted file mode 100644 index aaf4f293d7d8b..0000000000000 --- a/deps/checksums/SparseArrays-c9f7293c10e6bea775feceabd4554ab55c34e3a8.tar.gz/sha512 +++ /dev/null @@ -1 +0,0 @@ -37f5bca4d94bc74528079c039792017b7c363c53baf97671f458059f65134944e4c0ca87d474761ae082c8cf22090477559a40a84af53316aa117234d3145fa2 diff --git a/deps/checksums/SparseArrays-cb602d7b7cf46057ddc87d23cda2bdd168a548ac.tar.gz/md5 b/deps/checksums/SparseArrays-cb602d7b7cf46057ddc87d23cda2bdd168a548ac.tar.gz/md5 new file mode 100644 index 0000000000000..5234c20cb4ff7 --- /dev/null +++ b/deps/checksums/SparseArrays-cb602d7b7cf46057ddc87d23cda2bdd168a548ac.tar.gz/md5 @@ -0,0 +1 @@ +47cb7d9dd6f3d8ae3cb497c202ae6411 diff --git a/deps/checksums/SparseArrays-cb602d7b7cf46057ddc87d23cda2bdd168a548ac.tar.gz/sha512 b/deps/checksums/SparseArrays-cb602d7b7cf46057ddc87d23cda2bdd168a548ac.tar.gz/sha512 new file mode 100644 index 0000000000000..1aa2d8146d78e --- /dev/null +++ b/deps/checksums/SparseArrays-cb602d7b7cf46057ddc87d23cda2bdd168a548ac.tar.gz/sha512 @@ -0,0 +1 @@ +5ff47ea50564375e5e926c3f592a9708b1d9862e4090a53c6b02fc09bc1872578e016a4231564a10dd17be174beed54fd0b8821430828e7148f09556f8034ed9 diff --git a/stdlib/SparseArrays.version b/stdlib/SparseArrays.version index ec090d653f616..5eca7be5ec9b7 100644 --- a/stdlib/SparseArrays.version +++ b/stdlib/SparseArrays.version @@ -1,4 +1,4 @@ SPARSEARRAYS_BRANCH = main -SPARSEARRAYS_SHA1 = c9f7293c10e6bea775feceabd4554ab55c34e3a8 +SPARSEARRAYS_SHA1 = cb602d7b7cf46057ddc87d23cda2bdd168a548ac SPARSEARRAYS_GIT_URL := https://github.com/JuliaSparse/SparseArrays.jl.git SPARSEARRAYS_TAR_URL = https://api.github.com/repos/JuliaSparse/SparseArrays.jl/tarball/$1 From 913f7d584c602fbad5540f10c1c067d5ffac52b2 Mon Sep 17 00:00:00 2001 From: Shuhei Kadowaki <40514306+aviatesk@users.noreply.github.com> Date: Sun, 18 Feb 2024 02:41:58 +0900 Subject: [PATCH 02/25] allow external absint to hold custom data in `codeinst.inferred` (#53300) It has been possible for external abstract interpreters to keep custom data in `codeinst.inferred` (together /w overloading `inlining_policy`). After JuliaLang/julia#52233, when such external absint uses `InternalCodeCache`, this data is passed to `jl_ir_flag_inferred`, leading to segfaults in assertion builds. This commit resolves the issue by omitting `jl_ir_flag_inferred` checks when the `cache_owner` is external. Nonetheless, a better resolution might be necessary. It suggests that the current design of `code_owner` and `InternalCodeCache` for the external cache system is somewhat flawed. A conceivable approach could involve: - Adding a layer similar to `inlining_policy` in `CC.get(::WorldView{InternalCodeCache})` to enable safe redirection of custom data to the native interpreter's implementation. - Prohibiting custom data in the `inferred` field and directing such data to be kept in `analysis_results`. (cherry picked from commit 93876c92ed725272d8853d73dc0c3256095f1617) --- base/compiler/types.jl | 9 ++-- src/gf.c | 7 ++- test/compiler/AbstractInterpreter.jl | 33 +++++++++++- test/precompile.jl | 76 ++++++++++++++++++++++++++++ 4 files changed, 116 insertions(+), 9 deletions(-) diff --git a/base/compiler/types.jl b/base/compiler/types.jl index 642a7ac551662..53a363a2280e4 100644 --- a/base/compiler/types.jl +++ b/base/compiler/types.jl @@ -373,20 +373,17 @@ end function NativeInterpreter(world::UInt = get_world_counter(); inf_params::InferenceParams = InferenceParams(), opt_params::OptimizationParams = OptimizationParams()) + curr_max_world = get_world_counter() # Sometimes the caller is lazy and passes typemax(UInt). # we cap it to the current world age for correctness if world == typemax(UInt) - world = get_world_counter() + world = curr_max_world end - # If they didn't pass typemax(UInt) but passed something more subtly # incorrect, fail out loudly. - @assert world <= get_world_counter() - + @assert world <= curr_max_world method_table = CachedMethodTable(InternalMethodTable(world)) - inf_cache = Vector{InferenceResult}() # Initially empty cache - return NativeInterpreter(world, method_table, inf_cache, inf_params, opt_params) end diff --git a/src/gf.c b/src/gf.c index 20179157ff836..fdf61f440908b 100644 --- a/src/gf.c +++ b/src/gf.c @@ -444,8 +444,11 @@ STATIC_INLINE jl_value_t *_jl_rettype_inferred(jl_value_t *owner, jl_method_inst if (jl_atomic_load_relaxed(&codeinst->min_world) <= min_world && max_world <= jl_atomic_load_relaxed(&codeinst->max_world) && jl_egal(codeinst->owner, owner)) { - jl_value_t *code = jl_atomic_load_relaxed(&codeinst->inferred); - if (code && (code == jl_nothing || jl_ir_flag_inferred(code))) + jl_value_t *inferred = jl_atomic_load_relaxed(&codeinst->inferred); + if (inferred && ((inferred == jl_nothing) || ( + // allow whatever code instance external abstract interpreter produced + // since `jl_ir_flag_inferred` is specific to the native interpreter + codeinst->owner != jl_nothing || jl_ir_flag_inferred(inferred)))) return (jl_value_t*)codeinst; } codeinst = jl_atomic_load_relaxed(&codeinst->next); diff --git a/test/compiler/AbstractInterpreter.jl b/test/compiler/AbstractInterpreter.jl index c0b320009b8ec..2068997c77c82 100644 --- a/test/compiler/AbstractInterpreter.jl +++ b/test/compiler/AbstractInterpreter.jl @@ -399,7 +399,6 @@ end Core.eval(Core.Compiler, quote f(;a=1) = a end) @test_throws MethodError Core.Compiler.f(;b=2) - # Custom lookup function # ====================== @@ -469,3 +468,35 @@ let # generate cache @test occursin("j_sin_", s) @test !occursin("j_cos_", s) end + +# custom inferred data +# ==================== + +@newinterp CustomDataInterp +struct CustomDataInterpToken end +CC.cache_owner(::CustomDataInterp) = CustomDataInterpToken() +struct CustomData + inferred + CustomData(@nospecialize inferred) = new(inferred) +end +function CC.transform_result_for_cache(interp::CustomDataInterp, + mi::Core.MethodInstance, valid_worlds::CC.WorldRange, result::CC.InferenceResult) + inferred_result = @invoke CC.transform_result_for_cache(interp::CC.AbstractInterpreter, + mi::Core.MethodInstance, valid_worlds::CC.WorldRange, result::CC.InferenceResult) + return CustomData(inferred_result) +end +function CC.inlining_policy(interp::CustomDataInterp, @nospecialize(src), + @nospecialize(info::CC.CallInfo), stmt_flag::UInt32) + if src isa CustomData + src = src.inferred + end + return @invoke CC.inlining_policy(interp::CC.AbstractInterpreter, src::Any, + info::CC.CallInfo, stmt_flag::UInt32) +end +let src = code_typed((Int,); interp=CustomDataInterp()) do x + return sin(x) + cos(x) + end |> only |> first + @test count(isinvoke(:sin), src.code) == 1 + @test count(isinvoke(:cos), src.code) == 1 + @test count(isinvoke(:+), src.code) == 0 +end diff --git a/test/precompile.jl b/test/precompile.jl index c4cceced8235b..015cf49f228ac 100644 --- a/test/precompile.jl +++ b/test/precompile.jl @@ -1776,6 +1776,82 @@ let newinterp_path = abspath("compiler/newinterp.jl") @test found end end + + write(joinpath(load_path, "CustomAbstractInterpreterCaching2.jl"), :(module CustomAbstractInterpreterCaching2 + import SimpleModule: basic_caller, basic_callee + + module Custom + const CC = Core.Compiler + include("$($newinterp_path)") + @newinterp PrecompileInterpreter + struct CustomData + inferred + CustomData(@nospecialize inferred) = new(inferred) + end + function CC.transform_result_for_cache(interp::PrecompileInterpreter, + mi::Core.MethodInstance, valid_worlds::CC.WorldRange, result::CC.InferenceResult) + inferred_result = @invoke CC.transform_result_for_cache(interp::CC.AbstractInterpreter, + mi::Core.MethodInstance, valid_worlds::CC.WorldRange, result::CC.InferenceResult) + return CustomData(inferred_result) + end + function CC.inlining_policy(interp::PrecompileInterpreter, @nospecialize(src), + @nospecialize(info::CC.CallInfo), stmt_flag::UInt32) + if src isa CustomData + src = src.inferred + end + return @invoke CC.inlining_policy(interp::CC.AbstractInterpreter, src::Any, + info::CC.CallInfo, stmt_flag::UInt32) + end + end + + Base.return_types((Float64,)) do x + basic_caller(x) + end + Base.return_types((Float64,); interp=Custom.PrecompileInterpreter()) do x + basic_caller(x) + end + Base.return_types((Vector{Float64},)) do x + sum(x) + end + Base.return_types((Vector{Float64},); interp=Custom.PrecompileInterpreter()) do x + sum(x) + end + end) |> string) + Base.compilecache(Base.PkgId("CustomAbstractInterpreterCaching2")) + @eval let + using CustomAbstractInterpreterCaching2 + cache_owner = Core.Compiler.cache_owner( + CustomAbstractInterpreterCaching2.Custom.PrecompileInterpreter()) + let m = only(methods(CustomAbstractInterpreterCaching.basic_callee)) + mi = only(Base.specializations(m)) + ci = mi.cache + @test isdefined(ci, :next) + @test ci.owner === nothing + @test ci.max_world == typemax(UInt) + ci = ci.next + @test !isdefined(ci, :next) + @test ci.owner === cache_owner + @test ci.max_world == typemax(UInt) + end + let m = only(methods(sum, (Vector{Float64},))) + found = false + for mi = Base.specializations(m) + if mi isa Core.MethodInstance && mi.specTypes == Tuple{typeof(sum),Vector{Float64}} + ci = mi.cache + @test isdefined(ci, :next) + @test ci.owner === cache_owner + @test ci.max_world == typemax(UInt) + ci = ci.next + @test !isdefined(ci, :next) + @test ci.owner === nothing + @test ci.max_world == typemax(UInt) + found = true + break + end + end + @test found + end + end end end From 008d109d6d1a0da739170f72487344a8d60a0de8 Mon Sep 17 00:00:00 2001 From: Florian Atteneder Date: Thu, 15 Feb 2024 10:04:28 +0100 Subject: [PATCH 03/25] add Base.wrap to docs (cherry picked from commit 2197b5c9ed926c20ca03c6bf6fa903edd96ab6b1) --- doc/src/base/arrays.md | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/src/base/arrays.md b/doc/src/base/arrays.md index 20e8e81614b9e..f7ea23ad3b556 100644 --- a/doc/src/base/arrays.md +++ b/doc/src/base/arrays.md @@ -139,6 +139,7 @@ Base.reshape Base.dropdims Base.vec Base.SubArray +Base.wrap ``` ## Concatenation and permutation From 162bbfbceb1a9891c4bb37a7a2f6713d604e5396 Mon Sep 17 00:00:00 2001 From: Florian Atteneder Date: Sat, 17 Feb 2024 19:01:54 +0100 Subject: [PATCH 04/25] silence warnings in tests introduced in #33593 (cherry picked from commit 4990429a1f038ebeda573181609745bf30c4a7d6) --- test/file.jl | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/test/file.jl b/test/file.jl index 2dfd881b8a487..8f62b533083c6 100644 --- a/test/file.jl +++ b/test/file.jl @@ -637,9 +637,11 @@ end MAX_PATH = (Sys.iswindows() ? 260 - length(PATH_PREFIX) : 255) - 9 for i = 0:9 local tmp = joinpath(PATH_PREFIX, "x"^MAX_PATH * "123456789"[1:i]) - @test withenv(var => tmp) do - tempdir() - end == tmp + no_error_logging() do + @test withenv(var => tmp) do + tempdir() + end == tmp + end end end From 43dc98a85405f564715d9be17fee8e3f2893cef1 Mon Sep 17 00:00:00 2001 From: DilumAluthgeBot <43731525+DilumAluthgeBot@users.noreply.github.com> Date: Sun, 18 Feb 2024 21:07:49 -0500 Subject: [PATCH 05/25] =?UTF-8?q?=F0=9F=A4=96=20[master]=20Bump=20the=20Pk?= =?UTF-8?q?g=20stdlib=20from=206dd0e7c9e=20to=2076070d295=20(#53357)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Stdlib: Pkg URL: https://github.com/JuliaLang/Pkg.jl.git Stdlib branch: master Julia branch: master Old commit: 6dd0e7c9e New commit: 76070d295 Julia version: 1.12.0-DEV Pkg version: 1.11.0(Does not match) Bump invoked by: @IanButterworth Powered by: [BumpStdlibs.jl](https://github.com/JuliaLang/BumpStdlibs.jl) Diff: https://github.com/JuliaLang/Pkg.jl/compare/6dd0e7c9e99d578aa5477e2c78c91a161ce4c357...76070d295fc4a1f27f852e05400bbc956962e084 ``` $ git log --oneline 6dd0e7c9e..76070d295 76070d295 Prevent repl crash on invalid command (#3800) d267986c2 RFC: stop testing non-ARM on mac on CI (#3794) 2571be6d1 CI: Add Apple Silicon (macOS aarch64) to the CI matrix (#3793) 3075fb78c Simplify Pkg.Registry APIs. (#3785) e6f1e0902 Explain about Manifest.toml per julia version (#3791) ``` Co-authored-by: Dilum Aluthge (cherry picked from commit 59102aa21e1b37cd116f12553876937dd080dcb1) --- .../Pkg-6dd0e7c9e99d578aa5477e2c78c91a161ce4c357.tar.gz/md5 | 1 - .../Pkg-6dd0e7c9e99d578aa5477e2c78c91a161ce4c357.tar.gz/sha512 | 1 - .../Pkg-76070d295fc4a1f27f852e05400bbc956962e084.tar.gz/md5 | 1 + .../Pkg-76070d295fc4a1f27f852e05400bbc956962e084.tar.gz/sha512 | 1 + stdlib/Pkg.version | 2 +- 5 files changed, 3 insertions(+), 3 deletions(-) delete mode 100644 deps/checksums/Pkg-6dd0e7c9e99d578aa5477e2c78c91a161ce4c357.tar.gz/md5 delete mode 100644 deps/checksums/Pkg-6dd0e7c9e99d578aa5477e2c78c91a161ce4c357.tar.gz/sha512 create mode 100644 deps/checksums/Pkg-76070d295fc4a1f27f852e05400bbc956962e084.tar.gz/md5 create mode 100644 deps/checksums/Pkg-76070d295fc4a1f27f852e05400bbc956962e084.tar.gz/sha512 diff --git a/deps/checksums/Pkg-6dd0e7c9e99d578aa5477e2c78c91a161ce4c357.tar.gz/md5 b/deps/checksums/Pkg-6dd0e7c9e99d578aa5477e2c78c91a161ce4c357.tar.gz/md5 deleted file mode 100644 index 0bd59e0f13bab..0000000000000 --- a/deps/checksums/Pkg-6dd0e7c9e99d578aa5477e2c78c91a161ce4c357.tar.gz/md5 +++ /dev/null @@ -1 +0,0 @@ -fcf25a83bfb9c68eb34ecf50612ef2e5 diff --git a/deps/checksums/Pkg-6dd0e7c9e99d578aa5477e2c78c91a161ce4c357.tar.gz/sha512 b/deps/checksums/Pkg-6dd0e7c9e99d578aa5477e2c78c91a161ce4c357.tar.gz/sha512 deleted file mode 100644 index 03fc867f28070..0000000000000 --- a/deps/checksums/Pkg-6dd0e7c9e99d578aa5477e2c78c91a161ce4c357.tar.gz/sha512 +++ /dev/null @@ -1 +0,0 @@ -25fa3274513292c5e28502a8d9ffbf193ba44e32550e10dfcb880b7544780270ad155ef64d4346f9d10b8068fb8feb271c16a76aa93a99ede7f16ad5caafca38 diff --git a/deps/checksums/Pkg-76070d295fc4a1f27f852e05400bbc956962e084.tar.gz/md5 b/deps/checksums/Pkg-76070d295fc4a1f27f852e05400bbc956962e084.tar.gz/md5 new file mode 100644 index 0000000000000..279472ca7aa3d --- /dev/null +++ b/deps/checksums/Pkg-76070d295fc4a1f27f852e05400bbc956962e084.tar.gz/md5 @@ -0,0 +1 @@ +777fe3e3dd5b6c7c1b436d3077d91c13 diff --git a/deps/checksums/Pkg-76070d295fc4a1f27f852e05400bbc956962e084.tar.gz/sha512 b/deps/checksums/Pkg-76070d295fc4a1f27f852e05400bbc956962e084.tar.gz/sha512 new file mode 100644 index 0000000000000..5e26fbd8577fa --- /dev/null +++ b/deps/checksums/Pkg-76070d295fc4a1f27f852e05400bbc956962e084.tar.gz/sha512 @@ -0,0 +1 @@ +b832093b4e387460a1d96526169e12c6ee2ff5a8bd1961c362dbcf4f4839790f3a0bdcf1d0e1524b77eea662cc7590483fc7e4e674f94f5e7f291970778ab128 diff --git a/stdlib/Pkg.version b/stdlib/Pkg.version index 52524cb88083c..048b652a0a272 100644 --- a/stdlib/Pkg.version +++ b/stdlib/Pkg.version @@ -1,4 +1,4 @@ PKG_BRANCH = master -PKG_SHA1 = 6dd0e7c9e99d578aa5477e2c78c91a161ce4c357 +PKG_SHA1 = 76070d295fc4a1f27f852e05400bbc956962e084 PKG_GIT_URL := https://github.com/JuliaLang/Pkg.jl.git PKG_TAR_URL = https://api.github.com/repos/JuliaLang/Pkg.jl/tarball/$1 From d5dcb945979503c57ecf7a9ef94f4fea8b74581d Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Mon, 19 Feb 2024 07:52:29 -0500 Subject: [PATCH 06/25] fix sysimage-native-code=no option with pkgimages (#53373) Loading pkgimages would try to access the sysimage native code, which will fail. Ensure that no code tries to load if the sysimage native code is not available, as it may try to link against it. Fixes #53147 (cherry picked from commit 02699bb6cd83d16b1e51e2fb127241e18df7c56b) --- src/staticdata.c | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/src/staticdata.c b/src/staticdata.c index 62c0b27cb8eab..d7001382a2b10 100644 --- a/src/staticdata.c +++ b/src/staticdata.c @@ -607,15 +607,8 @@ extern void * JL_WEAK_SYMBOL_OR_ALIAS_DEFAULT(system_image_data_unavailable) jl_ extern void * JL_WEAK_SYMBOL_OR_ALIAS_DEFAULT(system_image_data_unavailable) jl_system_image_size; static void jl_load_sysimg_so(void) { - int imaging_mode = jl_generating_output() && !jl_options.incremental; - // in --build mode only use sysimg data, not precompiled native code - if (!imaging_mode && jl_options.use_sysimage_native_code==JL_OPTIONS_USE_SYSIMAGE_NATIVE_CODE_YES) { - assert(sysimage.fptrs.ptrs); - } - else { - memset(&sysimage.fptrs, 0, sizeof(sysimage.fptrs)); - } const char *sysimg_data; + assert(sysimage.fptrs.ptrs); // jl_init_processor_sysimg should already be run if (jl_sysimg_handle == jl_exe_handle && &jl_system_image_data != JL_WEAK_SYMBOL_DEFAULT(system_image_data_unavailable)) sysimg_data = (const char*)&jl_system_image_data; @@ -3097,6 +3090,17 @@ static void jl_restore_system_image_from_stream_(ios_t *f, jl_image_t *image, jl htable_new(&new_dt_objs, 0); arraylist_new(&deser_sym, 0); + // in --build mode only use sysimg data, not precompiled native code + int imaging_mode = jl_generating_output() && !jl_options.incremental; + if (!imaging_mode && jl_options.use_sysimage_native_code == JL_OPTIONS_USE_SYSIMAGE_NATIVE_CODE_YES) { + if (image->gvars_base) + assert(image->fptrs.ptrs); + } + else { + memset(&image->fptrs, 0, sizeof(image->fptrs)); + image->gvars_base = NULL; + } + // step 1: read section map assert(ios_pos(f) == 0 && f->bm == bm_mem); size_t sizeof_sysdata = read_uint(f); From f287e8ab4f7722fc80671c8efec0821d2df445fb Mon Sep 17 00:00:00 2001 From: tecosaur Date: Wed, 21 Feb 2024 01:21:58 +0800 Subject: [PATCH 07/25] More consistent return value for annotations, take 2 (#53333) Relands #53281 with some fixes noticed, though not the original causes of the failure. (cherry picked from commit fbc766aa9869e914e9540f5ad5b7ec03ccfcd47d) --- base/strings/annotated.jl | 27 ++++++++++++------- .../md5 | 1 - .../sha512 | 1 - .../md5 | 1 + .../sha512 | 1 + stdlib/StyledStrings.version | 2 +- 6 files changed, 20 insertions(+), 13 deletions(-) delete mode 100644 deps/checksums/StyledStrings-a1b2ae2434cd7d8199fa8647339422fe0e1d0324.tar.gz/md5 delete mode 100644 deps/checksums/StyledStrings-a1b2ae2434cd7d8199fa8647339422fe0e1d0324.tar.gz/sha512 create mode 100644 deps/checksums/StyledStrings-e0ca0f85412ea5cafabfeaaec4d62ca26c3959d2.tar.gz/md5 create mode 100644 deps/checksums/StyledStrings-e0ca0f85412ea5cafabfeaaec4d62ca26c3959d2.tar.gz/sha512 diff --git a/base/strings/annotated.jl b/base/strings/annotated.jl index 1eeaaa668d9ee..1b1130335f013 100644 --- a/base/strings/annotated.jl +++ b/base/strings/annotated.jl @@ -153,7 +153,7 @@ lastindex(s::AnnotatedString) = lastindex(s.string) function getindex(s::AnnotatedString, i::Integer) @boundscheck checkbounds(s, i) @inbounds if isvalid(s, i) - AnnotatedChar(s.string[i], annotations(s, i)) + AnnotatedChar(s.string[i], map(last, annotations(s, i))) else string_index_err(s, i) end @@ -356,37 +356,44 @@ annotate!(c::AnnotatedChar, @nospecialize(labelval::Pair{Symbol, <:Any})) = (push!(c.annotations, labelval); c) """ - annotations(str::AnnotatedString, [position::Union{Integer, UnitRange}]) - annotations(str::SubString{AnnotatedString}, [position::Union{Integer, UnitRange}]) + annotations(str::Union{AnnotatedString, SubString{AnnotatedString}}, + [position::Union{Integer, UnitRange}]) -> + Vector{Tuple{UnitRange{Int}, Pair{Symbol, Any}}} Get all annotations that apply to `str`. Should `position` be provided, only annotations that overlap with `position` will be returned. +Annotations are provided together with the regions they apply to, in the form of +a vector of region–annotation tuples. + See also: `annotate!`. """ annotations(s::AnnotatedString) = s.annotations -annotations(s::SubString{<:AnnotatedString}) = - annotations(s, s.offset+1:s.offset+s.ncodeunits) +function annotations(s::SubString{<:AnnotatedString}) + map(((region, annot),) -> (first(region)-s.offset:last(region)-s.offset, annot), + annotations(s.string, s.offset+1:s.offset+s.ncodeunits)) +end function annotations(s::AnnotatedString, pos::UnitRange{<:Integer}) # TODO optimise - annots = filter(label -> !isempty(intersect(pos, first(label))), - s.annotations) - last.(annots) + Tuple{UnitRange{Int64}, Pair{Symbol, Any}}[ + (max(first(pos), first(region)):min(last(pos), last(region)), annot) + for (region, annot) in s.annotations if !isempty(intersect(pos, region))] end annotations(s::AnnotatedString, pos::Integer) = annotations(s, pos:pos) annotations(s::SubString{<:AnnotatedString}, pos::Integer) = annotations(s.string, s.offset + pos) + annotations(s::SubString{<:AnnotatedString}, pos::UnitRange{<:Integer}) = annotations(s.string, first(pos)+s.offset:last(pos)+s.offset) """ - annotations(chr::AnnotatedChar) + annotations(chr::AnnotatedChar) -> Vector{Pair{Symbol, Any}} -Get all annotations of `chr`. +Get all annotations of `chr`, in the form of a vector of annotation pairs. """ annotations(c::AnnotatedChar) = c.annotations diff --git a/deps/checksums/StyledStrings-a1b2ae2434cd7d8199fa8647339422fe0e1d0324.tar.gz/md5 b/deps/checksums/StyledStrings-a1b2ae2434cd7d8199fa8647339422fe0e1d0324.tar.gz/md5 deleted file mode 100644 index fc675ee83125a..0000000000000 --- a/deps/checksums/StyledStrings-a1b2ae2434cd7d8199fa8647339422fe0e1d0324.tar.gz/md5 +++ /dev/null @@ -1 +0,0 @@ -c1fd1bb7e3f9ab00afe0758a8a101374 diff --git a/deps/checksums/StyledStrings-a1b2ae2434cd7d8199fa8647339422fe0e1d0324.tar.gz/sha512 b/deps/checksums/StyledStrings-a1b2ae2434cd7d8199fa8647339422fe0e1d0324.tar.gz/sha512 deleted file mode 100644 index 09d5a8a619295..0000000000000 --- a/deps/checksums/StyledStrings-a1b2ae2434cd7d8199fa8647339422fe0e1d0324.tar.gz/sha512 +++ /dev/null @@ -1 +0,0 @@ -373302f1d32a5b5ad0efacf14fa82a16fd877c11ca589fc1f1492eb1d4bcd0f49a2a5a3f471f8577649309c8a03634e7467e31b403cfd6ab8a8e4a775d7f4e53 diff --git a/deps/checksums/StyledStrings-e0ca0f85412ea5cafabfeaaec4d62ca26c3959d2.tar.gz/md5 b/deps/checksums/StyledStrings-e0ca0f85412ea5cafabfeaaec4d62ca26c3959d2.tar.gz/md5 new file mode 100644 index 0000000000000..2ab79799cca0e --- /dev/null +++ b/deps/checksums/StyledStrings-e0ca0f85412ea5cafabfeaaec4d62ca26c3959d2.tar.gz/md5 @@ -0,0 +1 @@ +fc3a846400107c432d20da6cfdd19ccf diff --git a/deps/checksums/StyledStrings-e0ca0f85412ea5cafabfeaaec4d62ca26c3959d2.tar.gz/sha512 b/deps/checksums/StyledStrings-e0ca0f85412ea5cafabfeaaec4d62ca26c3959d2.tar.gz/sha512 new file mode 100644 index 0000000000000..70b0ef6f5cb3a --- /dev/null +++ b/deps/checksums/StyledStrings-e0ca0f85412ea5cafabfeaaec4d62ca26c3959d2.tar.gz/sha512 @@ -0,0 +1 @@ +22da8964cc4c09f7c7a3da44be14c953f520ce6d395cf0f9ccf9c17777d6d968b0a874b35c072801ef7a1f4eee40f96ea0e2fc5ed5b3a63ad0b6b776a9c14ebb diff --git a/stdlib/StyledStrings.version b/stdlib/StyledStrings.version index 8180223575f05..19a5a24514f2f 100644 --- a/stdlib/StyledStrings.version +++ b/stdlib/StyledStrings.version @@ -1,4 +1,4 @@ STYLEDSTRINGS_BRANCH = main -STYLEDSTRINGS_SHA1 = a1b2ae2434cd7d8199fa8647339422fe0e1d0324 +STYLEDSTRINGS_SHA1 = e0ca0f85412ea5cafabfeaaec4d62ca26c3959d2 STYLEDSTRINGS_GIT_URL := https://github.com/JuliaLang/StyledStrings.jl.git STYLEDSTRINGS_TAR_URL = https://api.github.com/repos/JuliaLang/StyledStrings.jl/tarball/$1 From 6af35966451d5f2620ae5f9cb0c94a421e91a0a4 Mon Sep 17 00:00:00 2001 From: Jeff Bezanson Date: Tue, 20 Feb 2024 16:42:30 -0500 Subject: [PATCH 08/25] fix code coverage bug in tail position and `else` (#53354) This was due to lowering keeping the same location info for the inserted `return` or `goto` statement, even though the last seen location might not have executed. Also fixes inliner handling of the sentinel `0` value for code locations. (cherry picked from commit 61fc907a225eb642fd180257a02e5951336dabe4) --- base/compiler/ssair/inlining.jl | 4 +- base/compiler/ssair/passes.jl | 6 ++- src/julia-syntax.scm | 83 +++++++++++++++++++-------------- test/cmdlineargs.jl | 63 +++++++++++++++++++++++++ test/show.jl | 2 +- test/syntax.jl | 2 +- 6 files changed, 121 insertions(+), 39 deletions(-) diff --git a/base/compiler/ssair/inlining.jl b/base/compiler/ssair/inlining.jl index 74d3987ee0257..d4228cf3c4454 100644 --- a/base/compiler/ssair/inlining.jl +++ b/base/compiler/ssair/inlining.jl @@ -1818,7 +1818,9 @@ end function ssa_substitute!(insert_node!::Inserter, subst_inst::Instruction, @nospecialize(val), ssa_substitute::SSASubstitute) - subst_inst[:line] += ssa_substitute.linetable_offset + if subst_inst[:line] != 0 + subst_inst[:line] += ssa_substitute.linetable_offset + end return ssa_substitute_op!(insert_node!, subst_inst, val, ssa_substitute) end diff --git a/base/compiler/ssair/passes.jl b/base/compiler/ssair/passes.jl index fe587f82a9def..5ea21c85427be 100644 --- a/base/compiler/ssair/passes.jl +++ b/base/compiler/ssair/passes.jl @@ -1519,8 +1519,12 @@ function try_inline_finalizer!(ir::IRCode, argexprs::Vector{Any}, idx::Int, ssa_rename[ssa.id] end stmt′ = ssa_substitute_op!(InsertBefore(ir, SSAValue(idx)), inst, stmt′, ssa_substitute) + newline = inst[:line] + if newline != 0 + newline += ssa_substitute.linetable_offset + end ssa_rename[idx′] = insert_node!(ir, idx, - NewInstruction(inst; stmt=stmt′, line=inst[:line]+ssa_substitute.linetable_offset), + NewInstruction(inst; stmt=stmt′, line=newline), attach_after) end diff --git a/src/julia-syntax.scm b/src/julia-syntax.scm index 674a04cb1e563..7d9f8711ec714 100644 --- a/src/julia-syntax.scm +++ b/src/julia-syntax.scm @@ -4371,7 +4371,7 @@ f(x) = yt(x) (if (eq? (cdr s) dest-tokens) (cons (car s) l) (loop (cdr s) (cons (car s) l)))))) - (define (emit-return x) + (define (emit-return tail x) (define (emit- x) (let* ((tmp (if ((if (null? catch-token-stack) valid-ir-return? simple-atom?) x) #f @@ -4380,8 +4380,12 @@ f(x) = yt(x) (begin (emit `(= ,tmp ,x)) tmp) x))) (define (actually-return x) - (let* ((x (if rett - (compile (convert-for-type-decl (emit- x) rett #t lam) '() #t #f) + (let* ((x (begin0 (emit- x) + ;; if we are adding an implicit return then mark it as having no location + (if (not (eq? tail 'explicit)) + (emit '(line #f))))) + (x (if rett + (compile (convert-for-type-decl x rett #t lam) '() #t #f) x)) (x (emit- x))) (let ((pexc (pop-exc-expr catch-token-stack '()))) @@ -4531,7 +4535,7 @@ f(x) = yt(x) (eq? (car e) 'globalref)) (underscore-symbol? (cadr e))))) (error (string "all-underscore identifiers are write-only and their values cannot be used in expressions" (format-loc current-loc)))) - (cond (tail (emit-return e1)) + (cond (tail (emit-return tail e1)) (value e1) ((symbol? e1) (emit e1) #f) ;; keep symbols for undefined-var checking ((and (pair? e1) (eq? (car e1) 'outerref)) (emit e1) #f) ;; keep globals for undefined-var checking @@ -4577,7 +4581,7 @@ f(x) = yt(x) (else (compile-args (cdr e) break-labels)))) (callex (cons (car e) args))) - (cond (tail (emit-return callex)) + (cond (tail (emit-return tail callex)) (value callex) (else (emit callex))))) ((=) @@ -4594,7 +4598,7 @@ f(x) = yt(x) (if (not (eq? rr rhs)) (emit `(= ,rr ,rhs))) (emit `(= ,lhs ,rr)) - (if tail (emit-return rr)) + (if tail (emit-return tail rr)) rr) (emit-assignment lhs rhs)))))) ((block) @@ -4647,7 +4651,7 @@ f(x) = yt(x) (if file-diff (set! filename last-fname)) v))) ((return) - (compile (cadr e) break-labels #t #t) + (compile (cadr e) break-labels #t 'explicit) #f) ((unnecessary) ;; `unnecessary` marks expressions generated by lowering that @@ -4662,7 +4666,8 @@ f(x) = yt(x) (let ((v1 (compile (caddr e) break-labels value tail))) (if val (emit-assignment val v1)) (if (and (not tail) (or (length> e 3) val)) - (emit end-jump)) + (begin (emit `(line #f)) + (emit end-jump))) (let ((elselabel (make&mark-label))) (for-each (lambda (test) (set-car! (cddr test) elselabel)) @@ -4674,7 +4679,7 @@ f(x) = yt(x) (if (not tail) (set-car! (cdr end-jump) (make&mark-label)) (if (length= e 3) - (emit-return v2))) + (emit-return tail v2))) val)))) ((_while) (let* ((endl (make-label)) @@ -4716,7 +4721,7 @@ f(x) = yt(x) (emit `(label ,m)) (put! label-map (cadr e) (make&mark-label))) (if tail - (emit-return '(null)) + (emit-return tail '(null)) (if value (error "misplaced label"))))) ((symbolicgoto) (let* ((m (get label-map (cadr e) #f)) @@ -4762,7 +4767,7 @@ f(x) = yt(x) (begin (if els (begin (if (and (not val) v1) (emit v1)) (emit `(leave ,handler-token))) - (if v1 (emit-return v1))) + (if v1 (emit-return tail v1))) (if (not finally) (set! endl #f))) (begin (emit `(leave ,handler-token)) (emit `(goto ,(or els endl))))) @@ -4794,7 +4799,7 @@ f(x) = yt(x) (emit `(= ,tmp (call (core ===) ,finally ,(caar actions)))) (emit `(gotoifnot ,tmp ,skip)))) (let ((ac (cdar actions))) - (cond ((eq? (car ac) 'return) (emit-return (cadr ac))) + (cond ((eq? (car ac) 'return) (emit-return tail (cadr ac))) ((eq? (car ac) 'break) (emit-break (cadr ac))) (else ;; assumed to be a rethrow (emit ac)))) @@ -4833,8 +4838,8 @@ f(x) = yt(x) (set! global-const-error current-loc)) (emit e)))) ((atomic) (error "misplaced atomic declaration")) - ((isdefined) (if tail (emit-return e) e)) - ((boundscheck) (if tail (emit-return e) e)) + ((isdefined) (if tail (emit-return tail e) e)) + ((boundscheck) (if tail (emit-return tail e) e)) ((method) (if (not (null? (cadr lam))) @@ -4855,12 +4860,12 @@ f(x) = yt(x) l)))) (emit `(method ,(or (cadr e) '(false)) ,sig ,lam)) (if value (compile '(null) break-labels value tail))) - (cond (tail (emit-return e)) + (cond (tail (emit-return tail e)) (value e) (else (emit e))))) ((lambda) (let ((temp (linearize e))) - (cond (tail (emit-return temp)) + (cond (tail (emit-return tail temp)) (value temp) (else (emit temp))))) @@ -4868,7 +4873,7 @@ f(x) = yt(x) ((thunk module) (check-top-level e) (emit e) - (if tail (emit-return '(null))) + (if tail (emit-return tail '(null))) '(null)) ((toplevel-only) (check-top-level (cdr e)) @@ -4878,7 +4883,7 @@ f(x) = yt(x) (check-top-level e) (let ((val (make-ssavalue))) (emit `(= ,val ,e)) - (if tail (emit-return val)) + (if tail (emit-return tail val)) val)) ;; other top level expressions @@ -4887,7 +4892,7 @@ f(x) = yt(x) (emit e) (let ((have-ret? (and (pair? code) (pair? (car code)) (eq? (caar code) 'return)))) (if (and tail (not have-ret?)) - (emit-return '(null)))) + (emit-return tail '(null)))) '(null)) ((gc_preserve_begin) @@ -4911,7 +4916,7 @@ f(x) = yt(x) (else (emit e))) (if (and tail (not have-ret?)) - (emit-return '(null))) + (emit-return tail '(null))) '(null))) ;; unsupported assignment operators @@ -5027,6 +5032,7 @@ f(x) = yt(x) (labltable (table)) (ssavtable (table)) (current-loc 0) + (nowhere #f) (current-file file) (current-line line) (locstack '()) @@ -5040,26 +5046,33 @@ f(x) = yt(x) (set! current-loc 1))) (set! code (cons e code)) (set! i (+ i 1)) - (set! locs (cons current-loc locs))) + (set! locs (cons (if nowhere 0 current-loc) locs)) + (set! nowhere #f)) (let loop ((stmts (cdr body))) (if (pair? stmts) (let ((e (car stmts))) (cond ((atom? e) (emit e)) ((eq? (car e) 'line) - (if (and (= current-line 0) (length= e 2) (pair? linetable)) - ;; (line n) after push_loc just updates the line for the new file - (begin (set-lineno! (car linetable) (cadr e)) - (set! current-line (cadr e))) - (begin - (set! current-line (cadr e)) - (if (pair? (cddr e)) - (set! current-file (caddr e))) - (set! linetable (cons (if (null? locstack) - (make-lineinfo name current-file current-line) - (make-lineinfo name current-file current-line (caar locstack))) - linetable)) - (set! linetablelen (+ linetablelen 1)) - (set! current-loc linetablelen)))) + (cond ((and (length= e 2) (not (cadr e))) + ;; (line #f) marks that we are entering a generated statement + ;; that should not be counted as belonging to the previous marked location, + ;; for example `return` after a not-executed `if` arm in tail position. + (set! nowhere #t)) + ((and (= current-line 0) (length= e 2) (pair? linetable)) + ;; (line n) after push_loc just updates the line for the new file + (begin (set-lineno! (car linetable) (cadr e)) + (set! current-line (cadr e)))) + (else + (begin + (set! current-line (cadr e)) + (if (pair? (cddr e)) + (set! current-file (caddr e))) + (set! linetable (cons (if (null? locstack) + (make-lineinfo name current-file current-line) + (make-lineinfo name current-file current-line (caar locstack))) + linetable)) + (set! linetablelen (+ linetablelen 1)) + (set! current-loc linetablelen))))) ((and (length> e 2) (eq? (car e) 'meta) (eq? (cadr e) 'push_loc)) (set! locstack (cons (list current-loc current-line current-file) locstack)) (set! current-file (caddr e)) diff --git a/test/cmdlineargs.jl b/test/cmdlineargs.jl index 434f6be0d8b05..3af31965e63fc 100644 --- a/test/cmdlineargs.jl +++ b/test/cmdlineargs.jl @@ -545,6 +545,69 @@ let exename = `$(Base.julia_cmd()) --startup-file=no --color=no` got = read(covfile, String) @test isempty(got) rm(covfile) + + function coverage_info_for(src::String) + mktemp(dir) do srcfile, io + write(io, src); close(io) + outfile = tempname(dir, cleanup=false)*".info" + run(`$exename --code-coverage=$outfile $srcfile`) + result = read(outfile, String) + rm(outfile, force=true) + result + end + end + @test contains(coverage_info_for(""" + function cov_bug(x, p) + if p > 2 + print("") # runs + end + if Base.compilerbarrier(:const, false) + println("Does not run") + end + end + function do_test() + cov_bug(5, 3) + end + do_test() + """), """ + DA:1,1 + DA:2,1 + DA:3,1 + DA:5,1 + DA:6,0 + DA:9,1 + DA:10,1 + LH:6 + LF:7 + """) + @test contains(coverage_info_for(""" + function cov_bug() + if Base.compilerbarrier(:const, true) + if Base.compilerbarrier(:const, true) + if Base.compilerbarrier(:const, false) + println("Does not run") + end + else + print("Does not run either") + end + else + print("") + end + return nothing + end + cov_bug() + """), """ + DA:1,1 + DA:2,1 + DA:3,1 + DA:4,1 + DA:5,0 + DA:8,0 + DA:11,0 + DA:13,1 + LH:5 + LF:8 + """) end # --track-allocation diff --git a/test/show.jl b/test/show.jl index d6a691029d60a..66286f4df12ba 100644 --- a/test/show.jl +++ b/test/show.jl @@ -2106,7 +2106,7 @@ let src = code_typed(my_fun28173, (Int,), debuginfo=:source)[1][1] io = IOBuffer() Base.IRShow.show_ir(io, ir, Base.IRShow.default_config(ir; verbose_linetable=true)) seekstart(io) - @test count(contains(r"@ a{80}:\d+ within `my_fun28173"), eachline(io)) == 11 + @test count(contains(r"@ a{80}:\d+ within `my_fun28173"), eachline(io)) == 10 # Test that a bad :invoke doesn't cause an error during printing Core.Compiler.insert_node!(ir, 1, Core.Compiler.NewInstruction(Expr(:invoke, nothing, sin), Any), false) diff --git a/test/syntax.jl b/test/syntax.jl index a0021864da0e0..25d3e7282818a 100644 --- a/test/syntax.jl +++ b/test/syntax.jl @@ -713,7 +713,7 @@ m1_exprs = get_expr_list(Meta.lower(@__MODULE__, quote @m1 end)) let low3 = Meta.lower(@__MODULE__, quote @m3 end) m3_exprs = get_expr_list(low3) ci = low3.args[1]::Core.CodeInfo - @test ci.codelocs in ([4, 4, 2], [4, 2]) + @test ci.codelocs in ([4, 4, 0], [4, 0]) @test is_return_ssavalue(m3_exprs[end]) end From 2d6c154f6493c4b9458ec0e001cab16582f6338f Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Tue, 20 Feb 2024 23:07:24 -0500 Subject: [PATCH 09/25] fix sysimage-native-code=yes option (#53407) Follow up to #53373, it seems this assert was broken for empty packages, causing CI issues. It is not necessary. Observed in CI here: https://github.com/JuliaLang/julia/pull/53395 https://buildkite.com/julialang/julia-master/builds/33860#018dc4dc-a603-4ad1-90cf-574540a41720 (cherry picked from commit 3742d33ecc9048bb925ad89020751702c45f18a2) --- src/staticdata.c | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/src/staticdata.c b/src/staticdata.c index d7001382a2b10..63ebc1188c3ea 100644 --- a/src/staticdata.c +++ b/src/staticdata.c @@ -3092,11 +3092,7 @@ static void jl_restore_system_image_from_stream_(ios_t *f, jl_image_t *image, jl // in --build mode only use sysimg data, not precompiled native code int imaging_mode = jl_generating_output() && !jl_options.incremental; - if (!imaging_mode && jl_options.use_sysimage_native_code == JL_OPTIONS_USE_SYSIMAGE_NATIVE_CODE_YES) { - if (image->gvars_base) - assert(image->fptrs.ptrs); - } - else { + if (imaging_mode || jl_options.use_sysimage_native_code != JL_OPTIONS_USE_SYSIMAGE_NATIVE_CODE_YES) { memset(&image->fptrs, 0, sizeof(image->fptrs)); image->gvars_base = NULL; } From 2fdf5df48aa57c036c160f7e49fc737c99718ab5 Mon Sep 17 00:00:00 2001 From: Carsten Bauer Date: Wed, 21 Feb 2024 21:33:54 +0100 Subject: [PATCH 10/25] Fix documentation: thread pool of main thread (#53388) See https://github.com/JuliaLang/julia/issues/53217#issuecomment-1930891907 (cherry picked from commit 8425b0ea03a4cbf4cfb677fb554b19d6a5a0eb03) --- doc/src/manual/multi-threading.md | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/doc/src/manual/multi-threading.md b/doc/src/manual/multi-threading.md index 085c6b835c87c..dd02da8798f06 100644 --- a/doc/src/manual/multi-threading.md +++ b/doc/src/manual/multi-threading.md @@ -116,8 +116,8 @@ julia> using Base.Threads julia> nthreadpools() 2 -julia> threadpool() -:default +julia> threadpool() # the main thread is in the interactive thread pool +:interactive julia> nthreads(:default) 3 @@ -133,6 +133,10 @@ julia> nthreads() The zero-argument version of `nthreads` returns the number of threads in the default pool. +!!! note + Depending on whether Julia has been started with interactive threads, + the main thread is either in the default or interactive thread pool. + Either or both numbers can be replaced with the word `auto`, which causes Julia to choose a reasonable default. From 06e52c567d55d873c73e21097cef70a142cef413 Mon Sep 17 00:00:00 2001 From: Gabriel Baraldi Date: Thu, 22 Feb 2024 14:24:54 -0300 Subject: [PATCH 11/25] Fix synchronization issue on the GC scheduler (#53355) This aims to slightly simplify the synchronization by making `n_threads_marking` the sole memory location of relevance for it, it also removes the fast path, because being protected by the lock is quite important so that the observed gc state arrays are valid. Fixes: #53350 Fixes: #52757 Maybe fixes: #53026 Co-authored-by: Jameson Nash (cherry picked from commit a96726b84d9dfcaec304c5643956a2e9d2176079) --- src/gc.c | 63 ++++++++++++++------------------------- src/julia_threads.h | 11 +++---- src/llvm-codegen-shared.h | 2 +- src/safepoint.c | 4 +-- src/scheduler.c | 10 +++---- src/signal-handling.c | 2 +- src/threading.c | 2 +- 7 files changed, 37 insertions(+), 57 deletions(-) diff --git a/src/gc.c b/src/gc.c index a920958307c48..c2d194b8a28bb 100644 --- a/src/gc.c +++ b/src/gc.c @@ -3,6 +3,7 @@ #include "gc.h" #include "gc-page-profiler.h" #include "julia.h" +#include "julia_atomics.h" #include "julia_gcext.h" #include "julia_assert.h" #ifdef __GLIBC__ @@ -540,7 +541,7 @@ void jl_gc_run_all_finalizers(jl_task_t *ct) void jl_gc_add_finalizer_(jl_ptls_t ptls, void *v, void *f) JL_NOTSAFEPOINT { - assert(jl_atomic_load_relaxed(&ptls->gc_state) == 0); + assert(jl_atomic_load_relaxed(&ptls->gc_state) == JL_GC_STATE_UNSAFE); arraylist_t *a = &ptls->finalizers; // This acquire load and the release store at the end are used to // synchronize with `finalize_object` on another thread. Apart from the GC, @@ -1676,7 +1677,7 @@ void gc_sweep_wake_all(jl_ptls_t ptls, jl_gc_padded_page_stack_t *new_gc_allocd_ void gc_sweep_wait_for_all(void) { jl_atomic_store(&gc_allocd_scratch, NULL); - while (jl_atomic_load_relaxed(&gc_n_threads_sweeping) != 0) { + while (jl_atomic_load_acquire(&gc_n_threads_sweeping) != 0) { jl_cpu_pause(); } } @@ -2966,9 +2967,7 @@ void gc_mark_and_steal(jl_ptls_t ptls) jl_gc_markqueue_t *mq = &ptls->mark_queue; jl_gc_markqueue_t *mq_master = NULL; int master_tid = jl_atomic_load(&gc_master_tid); - if (master_tid == -1) { - return; - } + assert(master_tid != -1); mq_master = &gc_all_tls_states[master_tid]->mark_queue; void *new_obj; jl_gc_chunk_t c; @@ -3060,54 +3059,37 @@ size_t gc_count_work_in_queue(jl_ptls_t ptls) JL_NOTSAFEPOINT * Correctness argument for the mark-loop termination protocol. * * Safety properties: - * - No work items shall be in any thread's queues when `gc_mark_loop_barrier` observes + * - No work items shall be in any thread's queues when `gc_should_mark` observes * that `gc_n_threads_marking` is zero. * * - No work item shall be stolen from the master thread (i.e. mutator thread which started * GC and which helped the `jl_n_markthreads` - 1 threads to mark) after - * `gc_mark_loop_barrier` observes that `gc_n_threads_marking` is zero. This property is + * `gc_should_mark` observes that `gc_n_threads_marking` is zero. This property is * necessary because we call `gc_mark_loop_serial` after marking the finalizer list in * `_jl_gc_collect`, and want to ensure that we have the serial mark-loop semantics there, * and that no work is stolen from us at that point. * * Proof: - * - Suppose the master thread observes that `gc_n_threads_marking` is zero in - * `gc_mark_loop_barrier` and there is a work item left in one thread's queue at that point. - * Since threads try to steal from all threads' queues, this implies that all threads must - * have tried to steal from the queue which still has a work item left, but failed to do so, - * which violates the semantics of Chase-Lev's work-stealing queue. - * - * - Let E1 be the event "master thread writes -1 to gc_master_tid" and E2 be the event - * "master thread observes that `gc_n_threads_marking` is zero". Since we're using - * sequentially consistent atomics, E1 => E2. Now suppose one thread which is spinning in - * `gc_should_mark` tries to enter the mark-loop after E2. In order to do so, it must - * increment `gc_n_threads_marking` to 1 in an event E3, and then read `gc_master_tid` in an - * event E4. Since we're using sequentially consistent atomics, E3 => E4. Since we observed - * `gc_n_threads_marking` as zero in E2, then E2 => E3, and we conclude E1 => E4, so that - * the thread which is spinning in `gc_should_mark` must observe that `gc_master_tid` is -1 - * and therefore won't enter the mark-loop. + * - If a thread observes that `gc_n_threads_marking` is zero inside `gc_should_mark`, that + * means that no thread has work on their queue, this is guaranteed because a thread may only exit + * `gc_mark_and_steal` when its own queue is empty, this information is synchronized by the + * seq-cst fetch_add to a thread that is in `gc_should_mark`. `gc_queue_observer_lock` + * guarantees that once `gc_n_threads_marking` reaches zero, no thread will increment it again, + * because incrementing is only legal from inside the lock. Therefore, no thread will reenter + * the mark-loop after `gc_n_threads_marking` reaches zero. */ int gc_should_mark(void) { int should_mark = 0; - int n_threads_marking = jl_atomic_load(&gc_n_threads_marking); - // fast path - if (n_threads_marking == 0) { - return 0; - } uv_mutex_lock(&gc_queue_observer_lock); while (1) { - int tid = jl_atomic_load(&gc_master_tid); - // fast path - if (tid == -1) { - break; - } - n_threads_marking = jl_atomic_load(&gc_n_threads_marking); - // fast path + int n_threads_marking = jl_atomic_load(&gc_n_threads_marking); if (n_threads_marking == 0) { break; } + int tid = jl_atomic_load_relaxed(&gc_master_tid); + assert(tid != -1); size_t work = gc_count_work_in_queue(gc_all_tls_states[tid]); for (tid = gc_first_tid; tid < gc_first_tid + jl_n_markthreads; tid++) { jl_ptls_t ptls2 = gc_all_tls_states[tid]; @@ -3118,7 +3100,8 @@ int gc_should_mark(void) } // if there is a lot of work left, enter the mark loop if (work >= 16 * n_threads_marking) { - jl_atomic_fetch_add(&gc_n_threads_marking, 1); + jl_atomic_fetch_add(&gc_n_threads_marking, 1); // A possibility would be to allow a thread that found lots + // of work to increment this should_mark = 1; break; } @@ -3130,9 +3113,7 @@ int gc_should_mark(void) void gc_wake_all_for_marking(jl_ptls_t ptls) { - jl_atomic_store(&gc_master_tid, ptls->tid); uv_mutex_lock(&gc_threads_lock); - jl_atomic_fetch_add(&gc_n_threads_marking, 1); uv_cond_broadcast(&gc_threads_cond); uv_mutex_unlock(&gc_threads_lock); } @@ -3140,6 +3121,8 @@ void gc_wake_all_for_marking(jl_ptls_t ptls) void gc_mark_loop_parallel(jl_ptls_t ptls, int master) { if (master) { + jl_atomic_store(&gc_master_tid, ptls->tid); + jl_atomic_fetch_add(&gc_n_threads_marking, 1); gc_wake_all_for_marking(ptls); gc_mark_and_steal(ptls); jl_atomic_fetch_add(&gc_n_threads_marking, -1); @@ -3166,10 +3149,8 @@ void gc_mark_loop(jl_ptls_t ptls) void gc_mark_loop_barrier(void) { - jl_atomic_store(&gc_master_tid, -1); - while (jl_atomic_load(&gc_n_threads_marking) != 0) { - jl_cpu_pause(); - } + assert(jl_atomic_load_relaxed(&gc_n_threads_marking) == 0); + jl_atomic_store_relaxed(&gc_master_tid, -1); } void gc_mark_clean_reclaim_sets(void) diff --git a/src/julia_threads.h b/src/julia_threads.h index 1c64a722850f2..3f8f5391919b4 100644 --- a/src/julia_threads.h +++ b/src/julia_threads.h @@ -192,6 +192,9 @@ typedef struct _jl_tls_states_t { _Atomic(volatile size_t *) safepoint; // may be changed to the suspend page by any thread _Atomic(int8_t) sleep_check_state; // read/write from foreign threads // Whether it is safe to execute GC at the same time. +#define JL_GC_STATE_UNSAFE 0 + // gc_state = 0 means the thread is running Julia code and is not + // safe to run concurrently to the GC #define JL_GC_STATE_WAITING 1 // gc_state = 1 means the thread is doing GC or is waiting for the GC to // finish. @@ -330,9 +333,7 @@ STATIC_INLINE int8_t jl_gc_state_set(jl_ptls_t ptls, int8_t state, int8_t old_state) { jl_atomic_store_release(&ptls->gc_state, state); - if (state == JL_GC_STATE_SAFE && old_state == 0) - jl_gc_safepoint_(ptls); - if (state == 0 && old_state == JL_GC_STATE_SAFE) + if (state == JL_GC_STATE_UNSAFE || old_state == JL_GC_STATE_UNSAFE) jl_gc_safepoint_(ptls); return old_state; } @@ -347,8 +348,8 @@ void jl_gc_unsafe_leave(jl_ptls_t ptls, int8_t state) JL_NOTSAFEPOINT JL_NOTSAFE int8_t jl_gc_safe_enter(jl_ptls_t ptls) JL_NOTSAFEPOINT JL_NOTSAFEPOINT_ENTER; void jl_gc_safe_leave(jl_ptls_t ptls, int8_t state) JL_NOTSAFEPOINT_LEAVE; // this might not be a safepoint, but we have to assume it could be (statically) #else -#define jl_gc_unsafe_enter(ptls) jl_gc_state_save_and_set(ptls, 0) -#define jl_gc_unsafe_leave(ptls, state) ((void)jl_gc_state_set(ptls, (state), 0)) +#define jl_gc_unsafe_enter(ptls) jl_gc_state_save_and_set(ptls, JL_GC_STATE_UNSAFE) +#define jl_gc_unsafe_leave(ptls, state) ((void)jl_gc_state_set(ptls, (state), JL_GC_STATE_UNSAFE)) #define jl_gc_safe_enter(ptls) jl_gc_state_save_and_set(ptls, JL_GC_STATE_SAFE) #define jl_gc_safe_leave(ptls, state) ((void)jl_gc_state_set(ptls, (state), JL_GC_STATE_SAFE)) #endif diff --git a/src/llvm-codegen-shared.h b/src/llvm-codegen-shared.h index a4f77bc1b3b38..2c60907ca086c 100644 --- a/src/llvm-codegen-shared.h +++ b/src/llvm-codegen-shared.h @@ -325,7 +325,7 @@ static inline llvm::Value *emit_gc_unsafe_enter(llvm::IRBuilder<> &builder, llvm static inline llvm::Value *emit_gc_unsafe_leave(llvm::IRBuilder<> &builder, llvm::Type *T_size, llvm::Value *ptls, llvm::Value *state, bool final) { using namespace llvm; - Value *old_state = builder.getInt8(0); + Value *old_state = builder.getInt8(JL_GC_STATE_UNSAFE); return emit_gc_state_set(builder, T_size, ptls, state, old_state, final); } diff --git a/src/safepoint.c b/src/safepoint.c index a05d37e492813..22cda0a89444d 100644 --- a/src/safepoint.c +++ b/src/safepoint.c @@ -234,7 +234,7 @@ void jl_safepoint_wait_gc(void) JL_NOTSAFEPOINT jl_task_t *ct = jl_current_task; (void)ct; JL_TIMING_SUSPEND_TASK(GC_SAFEPOINT, ct); // The thread should have set this is already - assert(jl_atomic_load_relaxed(&ct->ptls->gc_state) != 0); + assert(jl_atomic_load_relaxed(&ct->ptls->gc_state) != JL_GC_STATE_UNSAFE); // Use normal volatile load in the loop for speed until GC finishes. // Then use an acquire load to make sure the GC result is visible on this thread. while (jl_atomic_load_relaxed(&jl_gc_running) || jl_atomic_load_acquire(&jl_gc_running)) { @@ -309,7 +309,7 @@ int jl_safepoint_suspend_thread(int tid, int waitstate) } while (jl_atomic_load_acquire(&ptls2->suspend_count) != 0) { int8_t state2 = jl_atomic_load_acquire(&ptls2->gc_state); - if (waitstate <= 2 && state2 != 0) + if (waitstate <= 2 && state2 != JL_GC_STATE_UNSAFE) break; if (waitstate == 3 && state2 == JL_GC_STATE_WAITING) break; diff --git a/src/scheduler.c b/src/scheduler.c index 2af88db89ca14..dbc913353ce4c 100644 --- a/src/scheduler.c +++ b/src/scheduler.c @@ -131,7 +131,7 @@ void jl_parallel_gc_threadfun(void *arg) jl_ptls_t ptls = jl_init_threadtls(targ->tid); // wait for all threads - jl_gc_state_set(ptls, JL_GC_STATE_WAITING, 0); + jl_gc_state_set(ptls, JL_GC_STATE_WAITING, JL_GC_STATE_UNSAFE); uv_barrier_wait(targ->barrier); // free the thread argument here @@ -143,9 +143,7 @@ void jl_parallel_gc_threadfun(void *arg) uv_cond_wait(&gc_threads_cond, &gc_threads_lock); } uv_mutex_unlock(&gc_threads_lock); - if (may_mark()) { - gc_mark_loop_parallel(ptls, 0); - } + gc_mark_loop_parallel(ptls, 0); if (may_sweep(ptls)) { // not an else! gc_sweep_pool_parallel(ptls); jl_atomic_fetch_add(&ptls->gc_sweeps_requested, -1); @@ -162,7 +160,7 @@ void jl_concurrent_gc_threadfun(void *arg) jl_ptls_t ptls = jl_init_threadtls(targ->tid); // wait for all threads - jl_gc_state_set(ptls, JL_GC_STATE_WAITING, 0); + jl_gc_state_set(ptls, JL_GC_STATE_WAITING, JL_GC_STATE_UNSAFE); uv_barrier_wait(targ->barrier); // free the thread argument here @@ -190,7 +188,7 @@ void jl_threadfun(void *arg) JL_GC_PROMISE_ROOTED(ct); // wait for all threads - jl_gc_state_set(ptls, JL_GC_STATE_SAFE, 0); + jl_gc_state_set(ptls, JL_GC_STATE_SAFE, JL_GC_STATE_UNSAFE); uv_barrier_wait(targ->barrier); // free the thread argument here diff --git a/src/signal-handling.c b/src/signal-handling.c index abe63ba6d2d7f..2ddbf2ad1cc8e 100644 --- a/src/signal-handling.c +++ b/src/signal-handling.c @@ -440,7 +440,7 @@ void jl_task_frame_noreturn(jl_task_t *ct) JL_NOTSAFEPOINT ct->ptls->in_finalizer = 0; ct->ptls->defer_signal = 0; // forcibly exit GC (if we were in it) or safe into unsafe, without the mandatory safepoint - jl_atomic_store_release(&ct->ptls->gc_state, 0); + jl_atomic_store_release(&ct->ptls->gc_state, JL_GC_STATE_UNSAFE); // allow continuing to use a Task that should have already died--unsafe necromancy! jl_atomic_store_relaxed(&ct->_state, JL_TASK_STATE_RUNNABLE); } diff --git a/src/threading.c b/src/threading.c index 648f7d935bac2..dcfc6ac7e93bb 100644 --- a/src/threading.c +++ b/src/threading.c @@ -364,7 +364,7 @@ jl_ptls_t jl_init_threadtls(int16_t tid) } } #endif - jl_atomic_store_relaxed(&ptls->gc_state, 0); // GC unsafe + jl_atomic_store_relaxed(&ptls->gc_state, JL_GC_STATE_UNSAFE); // GC unsafe // Conditionally initialize the safepoint address. See comment in // `safepoint.c` if (tid == 0) { From d57e7f778a430be1d00f048d7c679296630f7120 Mon Sep 17 00:00:00 2001 From: N5N3 <2642243996@qq.com> Date: Fri, 23 Feb 2024 07:06:06 +0800 Subject: [PATCH 12/25] =?UTF-8?q?Subtype:=20skip=20slow-path=20in=20`local?= =?UTF-8?q?=5F=E2=88=80=5F=E2=88=83=5Fsubtype`=20if=20inputs=20contain=20n?= =?UTF-8?q?o=20=E2=88=83=20typevar.=20(#53429)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This should be safe as ∀ vars' bounds are frozen in env. If there's no ∃ var, then the current env won't change after `local_∀_∃_subtype`. Thus, the slow path should be equivalent to the fast path if the latter returns 1. Close #53371. (cherry picked from commit 37c48e8b11d4742818571405019d80b59fceab49) --- src/jltypes.c | 2 +- src/julia_internal.h | 1 + src/subtype.c | 31 +++++++++++++++++++++++++++---- test/subtype.jl | 9 +++++++++ 4 files changed, 38 insertions(+), 5 deletions(-) diff --git a/src/jltypes.c b/src/jltypes.c index 9dac534259f73..e048b5a0f8dfd 100644 --- a/src/jltypes.c +++ b/src/jltypes.c @@ -217,7 +217,7 @@ JL_DLLEXPORT jl_array_t *jl_find_free_typevars(jl_value_t *v) } // test whether a type has vars bound by the given environment -static int jl_has_bound_typevars(jl_value_t *v, jl_typeenv_t *env) JL_NOTSAFEPOINT +int jl_has_bound_typevars(jl_value_t *v, jl_typeenv_t *env) JL_NOTSAFEPOINT { while (1) { if (jl_is_typevar(v)) { diff --git a/src/julia_internal.h b/src/julia_internal.h index 14b8b4f43ee92..2dc8fd0c105b6 100644 --- a/src/julia_internal.h +++ b/src/julia_internal.h @@ -744,6 +744,7 @@ void jl_method_table_activate(jl_methtable_t *mt, jl_typemap_entry_t *newentry); jl_typemap_entry_t *jl_method_table_add(jl_methtable_t *mt, jl_method_t *method, jl_tupletype_t *simpletype); jl_datatype_t *jl_mk_builtin_func(jl_datatype_t *dt, const char *name, jl_fptr_args_t fptr) JL_GC_DISABLED; int jl_obviously_unequal(jl_value_t *a, jl_value_t *b); +int jl_has_bound_typevars(jl_value_t *v, jl_typeenv_t *env) JL_NOTSAFEPOINT; JL_DLLEXPORT jl_array_t *jl_find_free_typevars(jl_value_t *v); int jl_has_fixed_layout(jl_datatype_t *t); JL_DLLEXPORT int jl_struct_try_layout(jl_datatype_t *dt); diff --git a/src/subtype.c b/src/subtype.c index 600df9da8757e..4a797aab9831d 100644 --- a/src/subtype.c +++ b/src/subtype.c @@ -1512,6 +1512,23 @@ static int may_contain_union_decision(jl_value_t *x, jl_stenv_t *e, jl_typeenv_t may_contain_union_decision(xb ? xb->ub : ((jl_tvar_t *)x)->ub, e, &newlog); } +static int has_exists_typevar(jl_value_t *x, jl_stenv_t *e) JL_NOTSAFEPOINT +{ + jl_typeenv_t *env = NULL; + jl_varbinding_t *v = e->vars; + while (v != NULL) { + if (v->right) { + jl_typeenv_t *newenv = (jl_typeenv_t*)alloca(sizeof(jl_typeenv_t)); + newenv->var = v->var; + newenv->val = NULL; + newenv->prev = env; + env = newenv; + } + v = v->prev; + } + return env != NULL && jl_has_bound_typevars(x, env); +} + static int local_forall_exists_subtype(jl_value_t *x, jl_value_t *y, jl_stenv_t *e, int param, int limit_slow) { int16_t oldRmore = e->Runions.more; @@ -1531,13 +1548,19 @@ static int local_forall_exists_subtype(jl_value_t *x, jl_value_t *y, jl_stenv_t int count = 0, noRmore = 0; sub = _forall_exists_subtype(x, y, e, param, &count, &noRmore); pop_unionstate(&e->Runions, &oldRunions); - // we should not try the slow path if `forall_exists_subtype` has tested all cases; - // Once limit_slow == 1, also skip it if - // 1) `forall_exists_subtype` return false + // We could skip the slow path safely if + // 1) `_∀_∃_subtype` has tested all cases + // 2) `_∀_∃_subtype` returns 1 && `x` and `y` contain no ∃ typevar + // Once `limit_slow == 1`, also skip it if + // 1) `_∀_∃_subtype` returns 0 // 2) the left `Union` looks big + // TODO: `limit_slow` ignores complexity from inner `local_∀_exists_subtype`. if (limit_slow == -1) limit_slow = kindx || kindy; - if (noRmore || (limit_slow && (count > 3 || !sub))) + int skip = noRmore || (limit_slow && (count > 3 || !sub)) || + (sub && (kindx || !has_exists_typevar(x, e)) && + (kindy || !has_exists_typevar(y, e))); + if (skip) e->Runions.more = oldRmore; } else { diff --git a/test/subtype.jl b/test/subtype.jl index c8197dbddbf6d..27e8fa604a5e9 100644 --- a/test/subtype.jl +++ b/test/subtype.jl @@ -2571,3 +2571,12 @@ let a = Tuple{Union{Nothing, Type{Pair{T1}} where T1}} b = Tuple{Type{X2} where X2<:(Pair{T2, Y2} where {Src, Z2<:Src, Y2<:Union{Val{Z2}, Z2}})} where T2 @test !Base.has_free_typevars(typeintersect(a, b)) end + +#issue 53371 +struct T53371{A,B,C,D,E} end +S53371{A} = Union{Int, <:A} +R53371{A} = Val{V} where V<:(T53371{B,C,D,E,F} where {B<:Val{A}, C<:S53371{B}, D<:S53371{B}, E<:S53371{B}, F<:S53371{B}}) +let S = Type{T53371{A, B, C, D, E}} where {A, B<:R53371{A}, C<:R53371{A}, D<:R53371{A}, E<:R53371{A}}, + T = Type{T53371{A, B, C, D, E} where {A, B<:R53371{A}, C<:R53371{A}, D<:R53371{A}, E<:R53371{A}}} + @test !(S <: T) +end From 63ebed78bcf498bbf78f8a020bf9d0fb6d0f57cc Mon Sep 17 00:00:00 2001 From: Max Horn Date: Fri, 23 Feb 2024 08:46:36 +0100 Subject: [PATCH 13/25] Add debug variant of loader_trampolines.o (#53437) This prevents a race condition when building 'julia-cli-debug julia-cli-release' simultaneously (as we do for libjulia_jll, and also generally seems appropriate given what is done for all other source files. Motivated by https://github.com/JuliaPackaging/Yggdrasil/pull/8151 so I'll first see if it works there. Closes #45002. (cherry picked from commit fee198beaba675772158ddc5faeeaf76beea612d) --- cli/Makefile | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/cli/Makefile b/cli/Makefile index c72ebff2b9bfd..4e32c53b9a6f0 100644 --- a/cli/Makefile +++ b/cli/Makefile @@ -48,7 +48,7 @@ LIB_DOBJS := $(BUILDDIR)/loader_lib.dbg.obj # If this is an architecture that supports dynamic linking, link in a trampoline definition ifneq (,$(wildcard $(SRCDIR)/trampolines/trampolines_$(ARCH).S)) LIB_OBJS += $(BUILDDIR)/loader_trampolines.o -LIB_DOBJS += $(BUILDDIR)/loader_trampolines.o +LIB_DOBJS += $(BUILDDIR)/loader_trampolines.dbg.obj endif default: release @@ -65,6 +65,8 @@ $(BUILDDIR)/loader_exe.dbg.obj : $(SRCDIR)/loader_exe.c $(HEADERS) $(JULIAHOME)/ @$(call PRINT_CC, $(CC) $(DEBUGFLAGS) $(LOADER_CFLAGS) -c $< -o $@) $(BUILDDIR)/loader_trampolines.o : $(SRCDIR)/trampolines/trampolines_$(ARCH).S $(HEADERS) $(SRCDIR)/trampolines/common.h @$(call PRINT_CC, $(CC) $(SHIPFLAGS) $(LOADER_CFLAGS) $< -c -o $@) +$(BUILDDIR)/loader_trampolines.dbg.obj : $(SRCDIR)/trampolines/trampolines_$(ARCH).S $(HEADERS) $(SRCDIR)/trampolines/common.h + @$(call PRINT_CC, $(CC) $(DEBUGFLAGS) $(LOADER_CFLAGS) $< -c -o $@) # Debugging target to help us see what kind of code is being generated for our trampolines dump-trampolines: $(SRCDIR)/trampolines/trampolines_$(ARCH).S From 99ad63e895f7df6391f725bcb946bff2ccbf72e6 Mon Sep 17 00:00:00 2001 From: tecosaur Date: Fri, 23 Feb 2024 16:16:42 +0800 Subject: [PATCH 14/25] Add annotate! method for AnnotatedIOBuffer (#53284) The annotate! function provides a convenient way of adding annotations to an AnnotatedString/AnnotatedChar without accessing any of the implementation details of the Annotated* types. When AnnotatedIOBuffer was introduced, an appropriate annotations method was added, but annotate! was missed. To correct that, we refactor the current annotate! method for AnnotatedString to a more general helper function _annotate! that operates on the annotation vector itself, and use this new helper method to easily provide an annotate! method for AnnotatedIOBuffer. (cherry picked from commit 6e1d062bccf0147b849461c7c09ddd03d94f71dc) --- base/strings/annotated.jl | 33 +++++++++++++++++++-------------- test/strings/annotated.jl | 6 ++++++ 2 files changed, 25 insertions(+), 14 deletions(-) diff --git a/base/strings/annotated.jl b/base/strings/annotated.jl index 1b1130335f013..a85cdf1b08bbb 100644 --- a/base/strings/annotated.jl +++ b/base/strings/annotated.jl @@ -316,28 +316,30 @@ reverse(s::SubString{<:AnnotatedString}) = reverse(AnnotatedString(s)) ## End AbstractString interface ## -""" - annotate!(str::AnnotatedString, [range::UnitRange{Int}], label::Symbol => value) - annotate!(str::SubString{AnnotatedString}, [range::UnitRange{Int}], label::Symbol => value) - -Annotate a `range` of `str` (or the entire string) with a labeled value (`label` => `value`). -To remove existing `label` annotations, use a value of `nothing`. -""" -function annotate!(s::AnnotatedString, range::UnitRange{Int}, @nospecialize(labelval::Pair{Symbol, <:Any})) +function _annotate!(annlist::Vector{Tuple{UnitRange{Int}, Pair{Symbol, Any}}}, range::UnitRange{Int}, @nospecialize(labelval::Pair{Symbol, <:Any})) label, val = labelval if val === nothing - indices = searchsorted(s.annotations, (range,), by=first) - labelindex = filter(i -> first(s.annotations[i][2]) === label, indices) + indices = searchsorted(annlist, (range,), by=first) + labelindex = filter(i -> first(annlist[i][2]) === label, indices) for index in Iterators.reverse(labelindex) - deleteat!(s.annotations, index) + deleteat!(annlist, index) end else - sortedindex = searchsortedlast(s.annotations, (range,), by=first) + 1 - insert!(s.annotations, sortedindex, (range, Pair{Symbol, Any}(label, val))) + sortedindex = searchsortedlast(annlist, (range,), by=first) + 1 + insert!(annlist, sortedindex, (range, Pair{Symbol, Any}(label, val))) end - s end +""" + annotate!(str::AnnotatedString, [range::UnitRange{Int}], label::Symbol => value) + annotate!(str::SubString{AnnotatedString}, [range::UnitRange{Int}], label::Symbol => value) + +Annotate a `range` of `str` (or the entire string) with a labeled value (`label` => `value`). +To remove existing `label` annotations, use a value of `nothing`. +""" +annotate!(s::AnnotatedString, range::UnitRange{Int}, @nospecialize(labelval::Pair{Symbol, <:Any})) = + (_annotate!(s.annotations, range, labelval); s) + annotate!(ss::AnnotatedString, @nospecialize(labelval::Pair{Symbol, <:Any})) = annotate!(ss, firstindex(ss):lastindex(ss), labelval) @@ -426,6 +428,9 @@ copy(io::AnnotatedIOBuffer) = AnnotatedIOBuffer(copy(io.io), copy(io.annotations annotations(io::AnnotatedIOBuffer) = io.annotations +annotate!(io::AnnotatedIOBuffer, range::UnitRange{Int}, @nospecialize(labelval::Pair{Symbol, <:Any})) = + (_annotate!(io.annotations, range, labelval); io) + function write(io::AnnotatedIOBuffer, astr::Union{AnnotatedString, SubString{<:AnnotatedString}}) astr = AnnotatedString(astr) offset = position(io.io) diff --git a/test/strings/annotated.jl b/test/strings/annotated.jl index 9d812deea51f8..fda583bf7f778 100644 --- a/test/strings/annotated.jl +++ b/test/strings/annotated.jl @@ -115,6 +115,12 @@ end @test write(aio, ' ') == 1 @test write(aio, Base.AnnotatedString("world", [(1:5, :tag => 2)])) == 5 @test Base.annotations(aio) == [(1:5, :tag => 1), (7:11, :tag => 2)] + # Check `annotate!`, including region sorting + @test truncate(aio, 0).io.size == 0 + @test write(aio, "hello world") == ncodeunits("hello world") + @test Base.annotate!(aio, 7:11, :tag => 2) === aio + @test Base.annotate!(aio, 1:5, :tag => 1) === aio + @test Base.annotations(aio) == [(1:5, :tag => 1), (7:11, :tag => 2)] # Reading @test read(seekstart(deepcopy(aio.io)), String) == "hello world" @test read(seekstart(deepcopy(aio)), String) == "hello world" From d49cdf51707e13c5b7acfc90c31020fee761a7b5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mos=C3=A8=20Giordano?= Date: Sun, 25 Feb 2024 22:55:35 +0000 Subject: [PATCH 15/25] [MozillaCACerts_jll] Update to v2023-12-12 (#53466) Memo to self: * update the version in `stdlib/MozillaCACerts_jll/Project.toml` * update `MOZILLA_CACERT_VERSION` in `deps/libgit2.version` * generate new checksums with `make -f contrib/refresh_checksums.mk mozillacert` * manually delete old checksums in `deps/checksums/cacert-` (cherry picked from commit 669cbdce4e40e3d06b168323b97c003d75483ca9) --- deps/checksums/cacert-2023-01-10.pem/md5 | 1 - deps/checksums/cacert-2023-01-10.pem/sha512 | 1 - deps/checksums/cacert-2023-12-12.pem/md5 | 1 + deps/checksums/cacert-2023-12-12.pem/sha512 | 1 + deps/libgit2.version | 3 ++- stdlib/MozillaCACerts_jll/Project.toml | 3 ++- 6 files changed, 6 insertions(+), 4 deletions(-) delete mode 100644 deps/checksums/cacert-2023-01-10.pem/md5 delete mode 100644 deps/checksums/cacert-2023-01-10.pem/sha512 create mode 100644 deps/checksums/cacert-2023-12-12.pem/md5 create mode 100644 deps/checksums/cacert-2023-12-12.pem/sha512 diff --git a/deps/checksums/cacert-2023-01-10.pem/md5 b/deps/checksums/cacert-2023-01-10.pem/md5 deleted file mode 100644 index 92063050b50f3..0000000000000 --- a/deps/checksums/cacert-2023-01-10.pem/md5 +++ /dev/null @@ -1 +0,0 @@ -e7cf471ba7c88f4e313f492a76e624b3 diff --git a/deps/checksums/cacert-2023-01-10.pem/sha512 b/deps/checksums/cacert-2023-01-10.pem/sha512 deleted file mode 100644 index d3322e5890f81..0000000000000 --- a/deps/checksums/cacert-2023-01-10.pem/sha512 +++ /dev/null @@ -1 +0,0 @@ -08cd35277bf2260cb3232d7a7ca3cce6b2bd58af9221922d2c6e9838a19c2f96d1ca6d77f3cc2a3ab611692f9fec939e9b21f67442282e867a487b0203ee0279 diff --git a/deps/checksums/cacert-2023-12-12.pem/md5 b/deps/checksums/cacert-2023-12-12.pem/md5 new file mode 100644 index 0000000000000..fa4b90b0b2ddd --- /dev/null +++ b/deps/checksums/cacert-2023-12-12.pem/md5 @@ -0,0 +1 @@ +1e305b4e910d204ab7b4c18ec0cf0f27 diff --git a/deps/checksums/cacert-2023-12-12.pem/sha512 b/deps/checksums/cacert-2023-12-12.pem/sha512 new file mode 100644 index 0000000000000..8ce26f24dffd9 --- /dev/null +++ b/deps/checksums/cacert-2023-12-12.pem/sha512 @@ -0,0 +1 @@ +dcbfe08d39efdd8de555b31e1050757900d08448c61f8a67c055f14514e5d47e734f6874b5ae628021c35ebc8dabdfdac5808df0b3a073d4f9246cca4b725fe8 diff --git a/deps/libgit2.version b/deps/libgit2.version index 9bd56f1bd0001..064c95094adb3 100644 --- a/deps/libgit2.version +++ b/deps/libgit2.version @@ -10,4 +10,5 @@ LIBGIT2_SHA1=a2bde63741977ca0f4ef7db2f609df320be67a08 # Specify the version of the Mozilla CA Certificate Store to obtain. # The versions of cacert.pem are identified by the date (YYYY-MM-DD) of their changes. # See https://curl.haxx.se/docs/caextract.html for more details. -MOZILLA_CACERT_VERSION := 2023-01-10 +# Keep in sync with `stdlib/MozillaCACerts_jll/Project.toml`. +MOZILLA_CACERT_VERSION := 2023-12-12 diff --git a/stdlib/MozillaCACerts_jll/Project.toml b/stdlib/MozillaCACerts_jll/Project.toml index cef860fda4acd..b4f667b974736 100644 --- a/stdlib/MozillaCACerts_jll/Project.toml +++ b/stdlib/MozillaCACerts_jll/Project.toml @@ -1,6 +1,7 @@ name = "MozillaCACerts_jll" uuid = "14a3606d-f60d-562e-9121-12d972cd8159" -version = "2023.01.10" +# Keep in sync with `deps/libgit2.version`. +version = "2023.12.12" [extras] Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" From 923789b7a617b0b3853e092165d873f92f5faf0d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mos=C3=A8=20Giordano?= Date: Mon, 26 Feb 2024 00:37:13 +0000 Subject: [PATCH 16/25] [LibGit2_jll] Update to v1.7.2 (#53467) Memo to self: * update version number in `stdlib/LibGit2_jll/Project.toml` * update test result in `stdlib/LibGit2_jll/test/runtests.jl` * update version number and commit sha in `deps/libgit2.version` * refresh checksums with `make -f contrib/refresh_checksums.mk -j libgit2` (cherry picked from commit 90d03b2bbec84488af7792258bf99fa2606b77a5) --- deps/checksums/libgit2 | 68 ++++++++++++++--------------- deps/libgit2.version | 4 +- stdlib/LibGit2_jll/Project.toml | 2 +- stdlib/LibGit2_jll/test/runtests.jl | 2 +- 4 files changed, 38 insertions(+), 38 deletions(-) diff --git a/deps/checksums/libgit2 b/deps/checksums/libgit2 index 63d67671b12a4..78b82572f3898 100644 --- a/deps/checksums/libgit2 +++ b/deps/checksums/libgit2 @@ -1,34 +1,34 @@ -LibGit2.v1.7.1+0.aarch64-apple-darwin.tar.gz/md5/80102fd8cd633a4875a1257bd61d4e17 -LibGit2.v1.7.1+0.aarch64-apple-darwin.tar.gz/sha512/3cc3679923c36e0020e692e79112a8fa71b53c1b83c9bea8d6defda124722a67c2859089d36fddef7be4547539575483db32de8137b43f7fc97843e579a02696 -LibGit2.v1.7.1+0.aarch64-linux-gnu.tar.gz/md5/74be95a3f7886a9804964f024df5311f -LibGit2.v1.7.1+0.aarch64-linux-gnu.tar.gz/sha512/3ad8a3c9ced9be2ab5fefe651f445a26900beae743127dcd1f887d01a7672d5d6c523641ba7d402620f3c44a1cc9557e43e11ad1692726c8cfabecca59a030e9 -LibGit2.v1.7.1+0.aarch64-linux-musl.tar.gz/md5/e63f4351250b4f4ac60d66b0bed2ddf5 -LibGit2.v1.7.1+0.aarch64-linux-musl.tar.gz/sha512/8f2dd17fe55b7cf7cf60504e5b630b22ce27e4e89d75f7e93dba3b112f662470612987e09abd82c2e3df48fc3c0fe1dbf98c690d972edb50c10a5571741cd9e8 -LibGit2.v1.7.1+0.armv6l-linux-gnueabihf.tar.gz/md5/f06611068a36fa575ec8eb219c068723 -LibGit2.v1.7.1+0.armv6l-linux-gnueabihf.tar.gz/sha512/04c554b9617300cea7750d18590e1a5629e70274ef1e1e8fdabbb4347e46fd8a68e82ba21714d7cd3809c3b6de2e254baca35ff60a4be87485643c460b10ac73 -LibGit2.v1.7.1+0.armv6l-linux-musleabihf.tar.gz/md5/7135ca6e52bf63855c5b6aa45d59ad80 -LibGit2.v1.7.1+0.armv6l-linux-musleabihf.tar.gz/sha512/e542180d2d8a0896ec586edac03b91d48d2ece3d22220d09b6e717b1b95a38bc1de2ae0faeed39dd3e99150684441bfb0504b55b3e18e543e00561f91147d405 -LibGit2.v1.7.1+0.armv7l-linux-gnueabihf.tar.gz/md5/7ffc92c821ec99bd76865ece43f5face -LibGit2.v1.7.1+0.armv7l-linux-gnueabihf.tar.gz/sha512/c945a0895be07479fd94c3c127060e58866bc9b9c93e80dc923ecdda6689e43d566896ecf4bfc7d85ca710b9ee51e6d85dec423edc48a3a2066db0fbb118d073 -LibGit2.v1.7.1+0.armv7l-linux-musleabihf.tar.gz/md5/3d00a6223612c23ac6b3c1f44bff8119 -LibGit2.v1.7.1+0.armv7l-linux-musleabihf.tar.gz/sha512/c3ef7783f140b28ad2d10e1c16f5be683d3332a4f9db5d26fdf3f2ac2b750aa0ceaec928740a9bdf7f4d9e83f666aa6e5fdd9c019630bf46f6797000943e1510 -LibGit2.v1.7.1+0.i686-linux-gnu.tar.gz/md5/6ea4e6777f5a8630f9fa98fb6a4a4ac7 -LibGit2.v1.7.1+0.i686-linux-gnu.tar.gz/sha512/d62a46b54dfc491a88fa35d06f3ed9c76ce676473b33acd5382e72ce07e6a313505755476c4732b7a22cd774ddcdf4ea5e8a5b62b93eb48b67363911655ac177 -LibGit2.v1.7.1+0.i686-linux-musl.tar.gz/md5/9f74dc4e93886f011412a4f61dfb487f -LibGit2.v1.7.1+0.i686-linux-musl.tar.gz/sha512/1370cab2ef917aa759dd4986f247a6b4878f12c4b17399fa75c9a2878f86c136e6f2f998a396df0757bf36ac09d5d194e4b7688705d115f09c176f4a5ab22347 -LibGit2.v1.7.1+0.i686-w64-mingw32.tar.gz/md5/ce866e600b2ad8c0fd54ff8c57dc015c -LibGit2.v1.7.1+0.i686-w64-mingw32.tar.gz/sha512/c7848b39f3515452e13fb156ee645f9a8d3917374ba874b10437b417b3c8e9a108e014b3baf30c7ced5fd0034d4f37de7e7d76fb105358d8e953dca30c873dc6 -LibGit2.v1.7.1+0.powerpc64le-linux-gnu.tar.gz/md5/b7c2f120e33f499860cb1e096923e7fe -LibGit2.v1.7.1+0.powerpc64le-linux-gnu.tar.gz/sha512/3df8e54d2086fbedb55b5dc31a2010f2ecd277089293473607e780340882bda5b2f9a2cc1c53c88bd7fcca0791cc2530645ceda17de3f37bb1ff98a19ccb85cf -LibGit2.v1.7.1+0.x86_64-apple-darwin.tar.gz/md5/b5334bd7e44c2c28705bb816fe03b9b7 -LibGit2.v1.7.1+0.x86_64-apple-darwin.tar.gz/sha512/d91cfde393499687cc699d55184c58ee5f543108902bf1f08fde2270dec0f38e0d70cbc7af04ffe46952afad12ce008e745f4aae9084f23df58982c14b48117c -LibGit2.v1.7.1+0.x86_64-linux-gnu.tar.gz/md5/9e2e2fe324a40bb0a5364d218c5ce45e -LibGit2.v1.7.1+0.x86_64-linux-gnu.tar.gz/sha512/da7e28c20c09c5c0731fd5cdff6fa6c319b2c4757d5c4228fc287238cd649f98c689814480119f21cbb938a29f52c895021b44c74eccc2f93ae51766555d9b6a -LibGit2.v1.7.1+0.x86_64-linux-musl.tar.gz/md5/7147480b9520116eb63ee3c30fa60a21 -LibGit2.v1.7.1+0.x86_64-linux-musl.tar.gz/sha512/f3dfb2a416cb786f229fe9eb3ef653a30ba5ebf3b978475f0a10fa79fa68b7bce9b6d99aed19f8dfb5599d988e3c6d4ede9ef1a6ccdbb3c2ea61f76b97d7fb29 -LibGit2.v1.7.1+0.x86_64-unknown-freebsd.tar.gz/md5/39e1a6d463e52ca0b2a1a8e6c3c4a286 -LibGit2.v1.7.1+0.x86_64-unknown-freebsd.tar.gz/sha512/3978ba9923cc8a188aca36d7320d46b2788de27142d11920976c47ad43574ad7056539812cebab62550e656b263c2d277754c341bd83d013de608a91e6a0aad3 -LibGit2.v1.7.1+0.x86_64-w64-mingw32.tar.gz/md5/7d92c546023f460741a8187999b76bbe -LibGit2.v1.7.1+0.x86_64-w64-mingw32.tar.gz/sha512/da00d54f969ce3b70cc95dda281ddfafee72073164c31d7999053ed704a59401d64894ad702306d6e19eb1a60e5e98e5960c9c7e9a0e1645a0f3048422e62eb9 -libgit2-e6325351ceee58cf56f58bdce61b38907805544f.tar.gz/md5/08777cc257825f218ceac1a24abafdc9 -libgit2-e6325351ceee58cf56f58bdce61b38907805544f.tar.gz/sha512/ebeaf3bb12ce7d58cd6d36e0123168de3af8f083f707dc20df9781537e38188a176667ac51daf8d9006d54f2beed13fbfff6c26fbb48e3228988578ef8fbc9b7 +LibGit2.v1.7.2+0.aarch64-apple-darwin.tar.gz/md5/069fa8cbd69e98a6196bd24bb427ff47 +LibGit2.v1.7.2+0.aarch64-apple-darwin.tar.gz/sha512/bacaf03e8aa28a55af2ae031f09a22f0a9cecb20257ee9778f37d87eb4ed21ff25dbb33ac342b134edc2a9584c0c57e6eabf7dbe78b031e9e45c2448bd8a317c +LibGit2.v1.7.2+0.aarch64-linux-gnu.tar.gz/md5/3bc5dfb163045c335aec05097702ddd4 +LibGit2.v1.7.2+0.aarch64-linux-gnu.tar.gz/sha512/c8055e51713f6fe6f064c7e4626db7a70d07af6e5c8cb031e88ef8ea728accdb92a449f738fb248c0afbc9698d30d0670fa5cf640df564886887c7ce4dbc7124 +LibGit2.v1.7.2+0.aarch64-linux-musl.tar.gz/md5/23440824a3ec0a0d82fa8adccac63534 +LibGit2.v1.7.2+0.aarch64-linux-musl.tar.gz/sha512/040874d092d7a11c5bfa284e95b86335792ffa6b868e8c7165ca304647d82ffbc7a72b65bb92038abd1bfa545e831d5b2a658b3f5891377735f85d3e4ddff7b2 +LibGit2.v1.7.2+0.armv6l-linux-gnueabihf.tar.gz/md5/3ffc0954875a20f610f6852c522311f2 +LibGit2.v1.7.2+0.armv6l-linux-gnueabihf.tar.gz/sha512/35767e10a36a0025539a354ef5be2ec4031457c0c1121a07f4616c774e48bb85425732935a12718d23627b4bdf1fb5c7fe6b220cb4e2bcb94b9d26c743af0004 +LibGit2.v1.7.2+0.armv6l-linux-musleabihf.tar.gz/md5/73acea595fc31967d7fe1e808f4cc47c +LibGit2.v1.7.2+0.armv6l-linux-musleabihf.tar.gz/sha512/e124681a7b7b1ff64a1ca073ee6fcc75fc2b11ab0a3404b0a2bc9944854c4048e576697254cd8efed1d6c7103ac2e1bd574a75c6c3d8a383775d48170ef6d474 +LibGit2.v1.7.2+0.armv7l-linux-gnueabihf.tar.gz/md5/b710035b0be0c7e4810b9282912369a0 +LibGit2.v1.7.2+0.armv7l-linux-gnueabihf.tar.gz/sha512/e34b2956a869303a5b6bed059f326989e4e95f039fa3d1b75fc6f80c6d1170df96e2ea809df05aad59685751eb4666a0a05e0c3b456bcace2b0570b174b2b541 +LibGit2.v1.7.2+0.armv7l-linux-musleabihf.tar.gz/md5/aa4c7374dda73266cd387596fa31c258 +LibGit2.v1.7.2+0.armv7l-linux-musleabihf.tar.gz/sha512/54f6c9bd7dd1ee0438b817b1b18a94ce6f18838613524f146cf37f7d01319dd7c05d19d352376454ed2658e9dd839fede32db05748d9c352b56c3f0fe01edebc +LibGit2.v1.7.2+0.i686-linux-gnu.tar.gz/md5/2cf407e8cf5f7c7c984b48704935afaa +LibGit2.v1.7.2+0.i686-linux-gnu.tar.gz/sha512/44224ccf773b9f77979c1130cbb893d80c934918940ab9357bff7d92b97bdf1671d4263d121a603a6c12e60375bafc38b4bcac9a12cb0abb4665b5848d12946a +LibGit2.v1.7.2+0.i686-linux-musl.tar.gz/md5/742b947992f9f003a49c480165df1c87 +LibGit2.v1.7.2+0.i686-linux-musl.tar.gz/sha512/4d1a0c5c119f391be76d93ee7e82c6e38c8cda5c172c3d431bed6c6923776d4ad194c88509203332bed23f4c42581255a319a2ca45b6433b3fb68b3dc0aa1f02 +LibGit2.v1.7.2+0.i686-w64-mingw32.tar.gz/md5/476c290758970376ddfdcad25ba74c7c +LibGit2.v1.7.2+0.i686-w64-mingw32.tar.gz/sha512/dc76e0563b45aead1187fd7abfe411e3fbc21ff45983b693c11db4265d39fc3f51d167fe447f6a7efb1f3ec1e35a9d0edb1c3444ed50be62770dfea545fbddae +LibGit2.v1.7.2+0.powerpc64le-linux-gnu.tar.gz/md5/3ce536ea1ad7facca14c19c84106e8e5 +LibGit2.v1.7.2+0.powerpc64le-linux-gnu.tar.gz/sha512/d52d400249c7ed16084b13fde81f784cbf3b13b7081944872986b216b7faf186b7c8ff28626504d461c6059c7b4962acde0b15dc2e36007492210568a0f425f7 +LibGit2.v1.7.2+0.x86_64-apple-darwin.tar.gz/md5/c59af177b7ebdcfd52312a40d7832ee8 +LibGit2.v1.7.2+0.x86_64-apple-darwin.tar.gz/sha512/5bae0efc71421f28e4a3b2aca70ef1810b5810ed3df3d3577b8a7e956ea895f4b9efdabf17a32130e1f71fede4ceae7246d099b71147686e5722abdf56d50778 +LibGit2.v1.7.2+0.x86_64-linux-gnu.tar.gz/md5/1a55eb14294730b0ba7a23df3aac27ee +LibGit2.v1.7.2+0.x86_64-linux-gnu.tar.gz/sha512/21f7aea927bed22da1b974eab4d6f3413cc41e47b5a5e5bbfff6690474d6f374952b81549345a68c9227d1f3ef9e76578c81776cd1c41e75330c9f2dd1437c7b +LibGit2.v1.7.2+0.x86_64-linux-musl.tar.gz/md5/33a9bfb5a805ca6ca695196c2e94183e +LibGit2.v1.7.2+0.x86_64-linux-musl.tar.gz/sha512/924c2c2cd9baad369b21c17364164dbff7ecda153cb1ed89a4a7539407f3bdc54e87304ed2b7e26ba8046077cb0b3645e9263d53eb4b220436a2e150a77109c3 +LibGit2.v1.7.2+0.x86_64-unknown-freebsd.tar.gz/md5/8d27555b9a1bc086c4c3056cda876bd2 +LibGit2.v1.7.2+0.x86_64-unknown-freebsd.tar.gz/sha512/8ca7a589a328908ee10490230c963ae1292f9cb7c2c427c26ca5d6e4985d56ef76d02f98e05e4fc655cc5e92e2e405d63896a180bc33ebd4b852d2ff355c0339 +LibGit2.v1.7.2+0.x86_64-w64-mingw32.tar.gz/md5/3d7ac5a2af376a2ba0b335cb84710874 +LibGit2.v1.7.2+0.x86_64-w64-mingw32.tar.gz/sha512/5bdbb1110fa7c5bfaac0ba63002f9b1f480abab1cf3e284146352c30027381c23919745a0c490e202e8d856503f5c2bb2390a6bc32af533def7d31f8d6f7be31 +libgit2-a418d9d4ab87bae16b87d8f37143a4687ae0e4b2.tar.gz/md5/c67e5a2358f01d26b78269639ba20148 +libgit2-a418d9d4ab87bae16b87d8f37143a4687ae0e4b2.tar.gz/sha512/c82dc18bacb45f82c72f09e5525fd47191c8673a553df9faa05290c33dfdadfff9784341e68790130a14b12267c795f4ca166481fdf9674d2b70e8228bbeb11d diff --git a/deps/libgit2.version b/deps/libgit2.version index 064c95094adb3..65f3e7670e4af 100644 --- a/deps/libgit2.version +++ b/deps/libgit2.version @@ -3,8 +3,8 @@ LIBGIT2_JLL_NAME := LibGit2 ## source build -LIBGIT2_BRANCH=v1.7.1 -LIBGIT2_SHA1=a2bde63741977ca0f4ef7db2f609df320be67a08 +LIBGIT2_BRANCH=v1.7.2 +LIBGIT2_SHA1=a418d9d4ab87bae16b87d8f37143a4687ae0e4b2 ## Other deps # Specify the version of the Mozilla CA Certificate Store to obtain. diff --git a/stdlib/LibGit2_jll/Project.toml b/stdlib/LibGit2_jll/Project.toml index e64d86dc6c9b7..1e13d0228488f 100644 --- a/stdlib/LibGit2_jll/Project.toml +++ b/stdlib/LibGit2_jll/Project.toml @@ -1,6 +1,6 @@ name = "LibGit2_jll" uuid = "e37daf67-58a4-590a-8e99-b0245dd2ffc5" -version = "1.7.1+0" +version = "1.7.2+0" [deps] MbedTLS_jll = "c8ffd9c3-330d-5841-b78e-0817d7145fa1" diff --git a/stdlib/LibGit2_jll/test/runtests.jl b/stdlib/LibGit2_jll/test/runtests.jl index 86dcf659e7ab2..6d83f6e447453 100644 --- a/stdlib/LibGit2_jll/test/runtests.jl +++ b/stdlib/LibGit2_jll/test/runtests.jl @@ -7,5 +7,5 @@ using Test, Libdl, LibGit2_jll minor = Ref{Cint}(0) patch = Ref{Cint}(0) @test ccall((:git_libgit2_version, libgit2), Cint, (Ref{Cint}, Ref{Cint}, Ref{Cint}), major, minor, patch) == 0 - @test VersionNumber(major[], minor[], patch[]) == v"1.7.1" + @test VersionNumber(major[], minor[], patch[]) == v"1.7.2" end From 103f96f410db6b109d912581bcde5aa2f3786739 Mon Sep 17 00:00:00 2001 From: Kristoffer Carlsson Date: Mon, 19 Feb 2024 13:11:08 +0100 Subject: [PATCH 17/25] add being able to pass a `CacheFlags()` to `isprecompiled` to verify a precompile file against a set of custom flags (#53332) (cherry picked from commit 9c0f1dc1e426008250415d5dd7aa155e9df1a75e) --- base/loading.jl | 268 ++++++++++++++++++++------------------ src/jl_exported_funcs.inc | 1 + src/staticdata.c | 2 +- src/staticdata_utils.c | 26 ++-- test/precompile.jl | 13 +- 5 files changed, 168 insertions(+), 142 deletions(-) diff --git a/base/loading.jl b/base/loading.jl index 8bcaa4b2ab5b3..f4db43d80640b 100644 --- a/base/loading.jl +++ b/base/loading.jl @@ -1495,6 +1495,126 @@ end # End extensions + +struct CacheFlags + # OOICCDDP - see jl_cache_flags + use_pkgimages::Bool + debug_level::Int + check_bounds::Int + inline::Bool + opt_level::Int +end +function CacheFlags(f::UInt8) + use_pkgimages = Bool(f & 1) + debug_level = Int((f >> 1) & 3) + check_bounds = Int((f >> 3) & 3) + inline = Bool((f >> 5) & 1) + opt_level = Int((f >> 6) & 3) # define OPT_LEVEL in statiddata_utils + CacheFlags(use_pkgimages, debug_level, check_bounds, inline, opt_level) +end +CacheFlags(f::Int) = CacheFlags(UInt8(f)) +CacheFlags() = CacheFlags(ccall(:jl_cache_flags, UInt8, ())) + +function _cacheflag_to_uint8(cf::CacheFlags)::UInt8 + f = UInt8(0) + f |= cf.use_pkgimages << 0 + f |= cf.debug_level << 1 + f |= cf.check_bounds << 3 + f |= cf.inline << 5 + f |= cf.opt_level << 6 + return f +end + +function show(io::IO, cf::CacheFlags) + print(io, "use_pkgimages = ", cf.use_pkgimages) + print(io, ", debug_level = ", cf.debug_level) + print(io, ", check_bounds = ", cf.check_bounds) + print(io, ", inline = ", cf.inline) + print(io, ", opt_level = ", cf.opt_level) +end + +struct ImageTarget + name::String + flags::Int32 + ext_features::String + features_en::Vector{UInt8} + features_dis::Vector{UInt8} +end + +function parse_image_target(io::IO) + flags = read(io, Int32) + nfeature = read(io, Int32) + feature_en = read(io, 4*nfeature) + feature_dis = read(io, 4*nfeature) + name_len = read(io, Int32) + name = String(read(io, name_len)) + ext_features_len = read(io, Int32) + ext_features = String(read(io, ext_features_len)) + ImageTarget(name, flags, ext_features, feature_en, feature_dis) +end + +function parse_image_targets(targets::Vector{UInt8}) + io = IOBuffer(targets) + ntargets = read(io, Int32) + targets = Vector{ImageTarget}(undef, ntargets) + for i in 1:ntargets + targets[i] = parse_image_target(io) + end + return targets +end + +function current_image_targets() + targets = @ccall jl_reflect_clone_targets()::Vector{UInt8} + return parse_image_targets(targets) +end + +struct FeatureName + name::Cstring + bit::UInt32 # bit index into a `uint32_t` array; + llvmver::UInt32 # 0 if it is available on the oldest LLVM version we support +end + +function feature_names() + fnames = Ref{Ptr{FeatureName}}() + nf = Ref{Csize_t}() + @ccall jl_reflect_feature_names(fnames::Ptr{Ptr{FeatureName}}, nf::Ptr{Csize_t})::Cvoid + if fnames[] == C_NULL + @assert nf[] == 0 + return Vector{FeatureName}(undef, 0) + end + Base.unsafe_wrap(Array, fnames[], nf[], own=false) +end + +function test_feature(features::Vector{UInt8}, feat::FeatureName) + bitidx = feat.bit + u8idx = div(bitidx, 8) + 1 + bit = bitidx % 8 + return (features[u8idx] & (1 << bit)) != 0 +end + +function show(io::IO, it::ImageTarget) + print(io, it.name) + if !isempty(it.ext_features) + print(io, ",", it.ext_features) + end + print(io, "; flags=", it.flags) + print(io, "; features_en=(") + first = true + for feat in feature_names() + if test_feature(it.features_en, feat) + name = Base.unsafe_string(feat.name) + if first + first = false + print(io, name) + else + print(io, ", ", name) + end + end + end + print(io, ")") + # Is feature_dis useful? +end + # should sync with the types of arguments of `stale_cachefile` const StaleCacheKey = Tuple{Base.PkgId, UInt128, String, String} @@ -1515,11 +1635,11 @@ function isprecompiled(pkg::PkgId; ignore_loaded::Bool=false, stale_cache::Dict{StaleCacheKey,Bool}=Dict{StaleCacheKey, Bool}(), cachepaths::Vector{String}=Base.find_all_in_cache_path(pkg), - sourcepath::Union{String,Nothing}=Base.locate_package(pkg) - ) + sourcepath::Union{String,Nothing}=Base.locate_package(pkg), + flags::CacheFlags=CacheFlags()) isnothing(sourcepath) && error("Cannot locate source for $(repr(pkg))") for path_to_try in cachepaths - staledeps = stale_cachefile(sourcepath, path_to_try, ignore_loaded = true) + staledeps = stale_cachefile(sourcepath, path_to_try, ignore_loaded = true, requested_flags=flags) if staledeps === true continue end @@ -1532,7 +1652,7 @@ function isprecompiled(pkg::PkgId; modpaths = find_all_in_cache_path(modkey) for modpath_to_try in modpaths::Vector{String} stale_cache_key = (modkey, modbuild_id, modpath, modpath_to_try)::StaleCacheKey - if get!(() -> stale_cachefile(stale_cache_key...; ignore_loaded) === true, + if get!(() -> stale_cachefile(stale_cache_key...; ignore_loaded, requested_flags=flags) === true, stale_cache, stale_cache_key) continue end @@ -2535,8 +2655,7 @@ function create_expr_cache(pkg::PkgId, input::String, output::String, output_o:: if output_o !== nothing @debug "Generating object cache file for $(repr("text/plain", pkg))" cpu_target = get(ENV, "JULIA_CPU_TARGET", nothing) - opt_level = Base.JLOptions().opt_level - opts = `-O$(opt_level) --output-o $(output_o) --output-ji $(output) --output-incremental=yes` + opts = `--output-o $(output_o) --output-ji $(output) --output-incremental=yes` else @debug "Generating cache file for $(repr("text/plain", pkg))" cpu_target = nothing @@ -2546,10 +2665,11 @@ function create_expr_cache(pkg::PkgId, input::String, output::String, output_o:: deps_eltype = sprint(show, eltype(concrete_deps); context = :module=>nothing) deps = deps_eltype * "[" * join(deps_strs, ",") * "]" trace = isassigned(PRECOMPILE_TRACE_COMPILE) ? `--trace-compile=$(PRECOMPILE_TRACE_COMPILE[])` : `` - io = open(pipeline(addenv(`$(julia_cmd(;cpu_target)::Cmd) $(opts) + io = open(pipeline(addenv(`$(julia_cmd(;cpu_target)::Cmd) + $(flags) + $(opts) --startup-file=no --history-file=no --warn-overwrite=yes --color=$(have_color === nothing ? "auto" : have_color ? "yes" : "no") - $flags $trace -`, "OPENBLAS_NUM_THREADS" => 1, @@ -3212,116 +3332,6 @@ function check_clone_targets(clone_targets) end end -struct CacheFlags - # OOICCDDP - see jl_cache_flags - use_pkgimages::Bool - debug_level::Int - check_bounds::Int - inline::Bool - opt_level::Int - - function CacheFlags(f::UInt8) - use_pkgimages = Bool(f & 1) - debug_level = Int((f >> 1) & 3) - check_bounds = Int((f >> 3) & 3) - inline = Bool((f >> 5) & 1) - opt_level = Int((f >> 6) & 3) # define OPT_LEVEL in statiddata_utils - new(use_pkgimages, debug_level, check_bounds, inline, opt_level) - end -end -CacheFlags(f::Int) = CacheFlags(UInt8(f)) -CacheFlags() = CacheFlags(ccall(:jl_cache_flags, UInt8, ())) - -function show(io::IO, cf::CacheFlags) - print(io, "use_pkgimages = ", cf.use_pkgimages) - print(io, ", debug_level = ", cf.debug_level) - print(io, ", check_bounds = ", cf.check_bounds) - print(io, ", inline = ", cf.inline) - print(io, ", opt_level = ", cf.opt_level) -end - -struct ImageTarget - name::String - flags::Int32 - ext_features::String - features_en::Vector{UInt8} - features_dis::Vector{UInt8} -end - -function parse_image_target(io::IO) - flags = read(io, Int32) - nfeature = read(io, Int32) - feature_en = read(io, 4*nfeature) - feature_dis = read(io, 4*nfeature) - name_len = read(io, Int32) - name = String(read(io, name_len)) - ext_features_len = read(io, Int32) - ext_features = String(read(io, ext_features_len)) - ImageTarget(name, flags, ext_features, feature_en, feature_dis) -end - -function parse_image_targets(targets::Vector{UInt8}) - io = IOBuffer(targets) - ntargets = read(io, Int32) - targets = Vector{ImageTarget}(undef, ntargets) - for i in 1:ntargets - targets[i] = parse_image_target(io) - end - return targets -end - -function current_image_targets() - targets = @ccall jl_reflect_clone_targets()::Vector{UInt8} - return parse_image_targets(targets) -end - -struct FeatureName - name::Cstring - bit::UInt32 # bit index into a `uint32_t` array; - llvmver::UInt32 # 0 if it is available on the oldest LLVM version we support -end - -function feature_names() - fnames = Ref{Ptr{FeatureName}}() - nf = Ref{Csize_t}() - @ccall jl_reflect_feature_names(fnames::Ptr{Ptr{FeatureName}}, nf::Ptr{Csize_t})::Cvoid - if fnames[] == C_NULL - @assert nf[] == 0 - return Vector{FeatureName}(undef, 0) - end - Base.unsafe_wrap(Array, fnames[], nf[], own=false) -end - -function test_feature(features::Vector{UInt8}, feat::FeatureName) - bitidx = feat.bit - u8idx = div(bitidx, 8) + 1 - bit = bitidx % 8 - return (features[u8idx] & (1 << bit)) != 0 -end - -function show(io::IO, it::ImageTarget) - print(io, it.name) - if !isempty(it.ext_features) - print(io, ",", it.ext_features) - end - print(io, "; flags=", it.flags) - print(io, "; features_en=(") - first = true - for feat in feature_names() - if test_feature(it.features_en, feat) - name = Base.unsafe_string(feat.name) - if first - first = false - print(io, name) - else - print(io, ", ", name) - end - end - end - print(io, ")") - # Is feature_dis useful? -end - # Set by FileWatching.__init__() global mkpidlock_hook global trymkpidlock_hook @@ -3377,11 +3387,11 @@ list_reasons(::Nothing) = "" # returns true if it "cachefile.ji" is stale relative to "modpath.jl" and build_id for modkey # otherwise returns the list of dependencies to also check -@constprop :none function stale_cachefile(modpath::String, cachefile::String; ignore_loaded::Bool = false, reasons=nothing) - return stale_cachefile(PkgId(""), UInt128(0), modpath, cachefile; ignore_loaded, reasons) +@constprop :none function stale_cachefile(modpath::String, cachefile::String; ignore_loaded::Bool = false, requested_flags::CacheFlags=CacheFlags(), reasons=nothing) + return stale_cachefile(PkgId(""), UInt128(0), modpath, cachefile; ignore_loaded, requested_flags, reasons) end @constprop :none function stale_cachefile(modkey::PkgId, build_id::UInt128, modpath::String, cachefile::String; - ignore_loaded::Bool = false, reasons::Union{Dict{String,Int},Nothing}=nothing) + ignore_loaded::Bool = false, requested_flags::CacheFlags=CacheFlags(), reasons::Union{Dict{String,Int},Nothing}=nothing) io = open(cachefile, "r") try checksum = isvalid_cache_header(io) @@ -3390,15 +3400,15 @@ end record_reason(reasons, "invalid header") return true # invalid cache file end - modules, (includes, _, requires), required_modules, srctextpos, prefs, prefs_hash, clone_targets, flags = parse_cache_header(io, cachefile) + modules, (includes, _, requires), required_modules, srctextpos, prefs, prefs_hash, clone_targets, actual_flags = parse_cache_header(io, cachefile) if isempty(modules) return true # ignore empty file end - if ccall(:jl_match_cache_flags, UInt8, (UInt8,), flags) == 0 + if @ccall(jl_match_cache_flags(_cacheflag_to_uint8(requested_flags)::UInt8, actual_flags::UInt8)::UInt8) == 0 @debug """ Rejecting cache file $cachefile for $modkey since the flags are mismatched - current session: $(CacheFlags()) - cache file: $(CacheFlags(flags)) + requested flags: $(requested_flags) [$(_cacheflag_to_uint8(requested_flags))] + cache file: $(CacheFlags(actual_flags)) [$actual_flags] """ record_reason(reasons, "mismatched flags") return true @@ -3672,7 +3682,7 @@ function precompile(@nospecialize(argt::Type), m::Method) return precompile(mi) end -precompile(include_package_for_output, (PkgId, String, Vector{String}, Vector{String}, Vector{String}, typeof(_concrete_dependencies), Nothing)) -precompile(include_package_for_output, (PkgId, String, Vector{String}, Vector{String}, Vector{String}, typeof(_concrete_dependencies), String)) -precompile(create_expr_cache, (PkgId, String, String, String, typeof(_concrete_dependencies), IO, IO, Cmd)) -precompile(create_expr_cache, (PkgId, String, String, Nothing, typeof(_concrete_dependencies), IO, IO, Cmd)) +@assert precompile(include_package_for_output, (PkgId, String, Vector{String}, Vector{String}, Vector{String}, typeof(_concrete_dependencies), Nothing)) +@assert precompile(include_package_for_output, (PkgId, String, Vector{String}, Vector{String}, Vector{String}, typeof(_concrete_dependencies), String)) +@assert precompile(create_expr_cache, (PkgId, String, String, String, typeof(_concrete_dependencies), Cmd, IO, IO)) +@assert precompile(create_expr_cache, (PkgId, String, String, Nothing, typeof(_concrete_dependencies), Cmd, IO, IO)) diff --git a/src/jl_exported_funcs.inc b/src/jl_exported_funcs.inc index 91f989f611b22..de1bc7601f60d 100644 --- a/src/jl_exported_funcs.inc +++ b/src/jl_exported_funcs.inc @@ -387,6 +387,7 @@ XX(jl_raise_debugger) \ XX(jl_readuntil) \ XX(jl_cache_flags) \ + XX(jl_match_cache_flags_current) \ XX(jl_match_cache_flags) \ XX(jl_read_verify_header) \ XX(jl_realloc) \ diff --git a/src/staticdata.c b/src/staticdata.c index 63ebc1188c3ea..f899899aff242 100644 --- a/src/staticdata.c +++ b/src/staticdata.c @@ -3566,7 +3566,7 @@ static jl_value_t *jl_validate_cache_file(ios_t *f, jl_array_t *depmods, uint64_ "Precompile file header verification checks failed."); } uint8_t flags = read_uint8(f); - if (pkgimage && !jl_match_cache_flags(flags)) { + if (pkgimage && !jl_match_cache_flags_current(flags)) { return jl_get_exceptionf(jl_errorexception_type, "Pkgimage flags mismatch"); } if (!pkgimage) { diff --git a/src/staticdata_utils.c b/src/staticdata_utils.c index fc7a59d9d0bf9..9542e54182644 100644 --- a/src/staticdata_utils.c +++ b/src/staticdata_utils.c @@ -603,31 +603,35 @@ JL_DLLEXPORT uint8_t jl_cache_flags(void) return flags; } -JL_DLLEXPORT uint8_t jl_match_cache_flags(uint8_t flags) + +JL_DLLEXPORT uint8_t jl_match_cache_flags(uint8_t requested_flags, uint8_t actual_flags) { - // 1. Check which flags are relevant - uint8_t current_flags = jl_cache_flags(); - uint8_t supports_pkgimage = (current_flags & 1); - uint8_t is_pkgimage = (flags & 1); + uint8_t supports_pkgimage = (requested_flags & 1); + uint8_t is_pkgimage = (actual_flags & 1); // For .ji packages ignore other flags if (!supports_pkgimage && !is_pkgimage) { return 1; } - // If package images are optional, ignore that bit (it will be unset in current_flags) + // If package images are optional, ignore that bit (it will be unset in requested_flags) if (jl_options.use_pkgimages == JL_OPTIONS_USE_PKGIMAGES_EXISTING) { - flags &= ~1; + actual_flags &= ~1; } // 2. Check all flags, execept opt level must be exact uint8_t mask = (1 << OPT_LEVEL)-1; - if ((flags & mask) != (current_flags & mask)) + if ((actual_flags & mask) != (requested_flags & mask)) return 0; // 3. allow for higher optimization flags in cache - flags >>= OPT_LEVEL; - current_flags >>= OPT_LEVEL; - return flags >= current_flags; + actual_flags >>= OPT_LEVEL; + requested_flags >>= OPT_LEVEL; + return actual_flags >= requested_flags; +} + +JL_DLLEXPORT uint8_t jl_match_cache_flags_current(uint8_t flags) +{ + return jl_match_cache_flags(jl_cache_flags(), flags); } // return char* from String field in Base.GIT_VERSION_INFO diff --git a/test/precompile.jl b/test/precompile.jl index 015cf49f228ac..a68d8936d1ed1 100644 --- a/test/precompile.jl +++ b/test/precompile.jl @@ -2072,8 +2072,16 @@ precompile_test_harness("Test flags") do load_path module TestFlags end """) + + current_flags = Base.CacheFlags() + modified_flags = Base.CacheFlags( + current_flags.use_pkgimages, + current_flags.debug_level, + 2, + current_flags.inline, + 3 + ) ji, ofile = Base.compilecache(Base.PkgId("TestFlags"); flags=`--check-bounds=no -O3`) - @show ji, ofile open(ji, "r") do io Base.isvalid_cache_header(io) _, _, _, _, _, _, _, flags = Base.parse_cache_header(io, ji) @@ -2081,6 +2089,9 @@ precompile_test_harness("Test flags") do load_path @test cacheflags.check_bounds == 2 @test cacheflags.opt_level == 3 end + id = Base.identify_package("TestFlags") + @test Base.isprecompiled(id, ;flags=modified_flags) + @test !Base.isprecompiled(id, ;flags=current_flags) end empty!(Base.DEPOT_PATH) From 00cbe4acb243d43ac4acb1ca9b2f048427abbe3a Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Fri, 23 Feb 2024 14:59:20 -0500 Subject: [PATCH 18/25] when loading code for internal purposes, load stdlib files directly(#53326) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This bypasses DEPOT_PATH, LOAD_PATH, and stale checks when loading known stdlib code for known purposes from known locations, specifically to avoid the problem that I cannot fix my tools because I have used my tools to break my tools. This helps avoid the situation that the user can break important Pkg, REPL, Socket or similar features simply because they chose to remove `@stdlib` from their environment. For example, if you make an edit to REPL, then this will trigger recompilation and load the edited version: ``` $ ./julia -e 'using REPL' -iq ┌ Info: Precompiling REPL [3fa0cd96-eef1-5676-8a61-b3b8758bbffb] (cache misses: include_dependency fsize change (2), invalid header (10), mismatched flags (1)) └ @ Base loading.jl:2643 julia> ``` But this will load the version that shipped with Julia, regardless of the state of the cache or the source code (unless you delete the cache files): ``` $ ./julia -iq julia> ``` Fixes #53365 (cherry picked from commit ea1a0d2512be72a4568348e6b5fdfbbe687436c3) --- base/Base.jl | 3 +- base/client.jl | 7 +- base/loading.jl | 311 +++++++++++------- base/stream.jl | 4 +- base/util.jl | 2 +- .../InteractiveUtils/src/InteractiveUtils.jl | 4 +- stdlib/LinearAlgebra/src/LinearAlgebra.jl | 2 +- stdlib/REPL/src/REPL.jl | 22 +- test/precompile.jl | 6 +- 9 files changed, 210 insertions(+), 151 deletions(-) diff --git a/base/Base.jl b/base/Base.jl index c729ab9ca707e..e02b132aae6e4 100644 --- a/base/Base.jl +++ b/base/Base.jl @@ -629,8 +629,7 @@ function profile_printing_listener(cond::Base.AsyncCondition) profile = nothing try while _trywait(cond) - # this call to require is mostly legal, only because Profile has no dependencies and is usually in LOAD_PATH - profile = @something(profile, require(PkgId(UUID("9abbd945-dff8-562f-b5e8-e1ebf5ef1b79"), "Profile")))::Module + profile = @something(profile, require_stdlib(PkgId(UUID("9abbd945-dff8-562f-b5e8-e1ebf5ef1b79"), "Profile")))::Module invokelatest(profile.peek_report[]) if Base.get_bool_env("JULIA_PROFILE_PEEK_HEAP_SNAPSHOT", false) === true println(stderr, "Saving heap snapshot...") diff --git a/base/client.jl b/base/client.jl index 201792c786b51..31a3016489dcd 100644 --- a/base/client.jl +++ b/base/client.jl @@ -260,7 +260,7 @@ function exec_options(opts) # Load Distributed module only if any of the Distributed options have been specified. distributed_mode = (opts.worker == 1) || (opts.nprocs > 0) || (opts.machine_file != C_NULL) if distributed_mode - let Distributed = require(PkgId(UUID((0x8ba89e20_285c_5b6f, 0x9357_94700520ee1b)), "Distributed")) + let Distributed = require_stdlib(PkgId(UUID((0x8ba89e20_285c_5b6f, 0x9357_94700520ee1b)), "Distributed")) Core.eval(MainInclude, :(const Distributed = $Distributed)) Core.eval(Main, :(using Base.MainInclude.Distributed)) end @@ -386,7 +386,8 @@ function load_InteractiveUtils(mod::Module=Main) # load interactive-only libraries if !isdefined(MainInclude, :InteractiveUtils) try - let InteractiveUtils = require(PkgId(UUID(0xb77e0a4c_d291_57a0_90e8_8db25a27a240), "InteractiveUtils")) + # TODO: we have to use require_stdlib here because it is a dependency of REPL, but we would sort of prefer not to + let InteractiveUtils = require_stdlib(PkgId(UUID(0xb77e0a4c_d291_57a0_90e8_8db25a27a240), "InteractiveUtils")) Core.eval(MainInclude, :(const InteractiveUtils = $InteractiveUtils)) end catch ex @@ -401,7 +402,7 @@ end function load_REPL() # load interactive-only libraries try - return Base.require(PkgId(UUID(0x3fa0cd96_eef1_5676_8a61_b3b8758bbffb), "REPL")) + return Base.require_stdlib(PkgId(UUID(0x3fa0cd96_eef1_5676_8a61_b3b8758bbffb), "REPL")) catch ex @warn "Failed to import REPL" exception=(ex, catch_backtrace()) end diff --git a/base/loading.jl b/base/loading.jl index f4db43d80640b..0ec9342582db5 100644 --- a/base/loading.jl +++ b/base/loading.jl @@ -256,9 +256,9 @@ struct LoadingCache env_project_file::Dict{String, Union{Bool, String}} project_file_manifest_path::Dict{String, Union{Nothing, String}} require_parsed::Set{String} - identified_where::Dict{Tuple{PkgId, String}, Union{Nothing, Tuple{PkgId, Union{Nothing, String}}}} - identified::Dict{String, Union{Nothing, Tuple{PkgId, Union{Nothing, String}}}} - located::Dict{Tuple{PkgId, Union{String, Nothing}}, Union{Tuple{Union{String, Nothing}, Union{String, Nothing}}, Nothing}} + identified_where::Dict{Tuple{PkgId, String}, Union{Nothing, Tuple{PkgId, String}}} + identified::Dict{String, Union{Nothing, Tuple{PkgId, String}}} + located::Dict{Tuple{PkgId, Union{String, Nothing}}, Union{Tuple{String, String}, Nothing}} end const LOADING_CACHE = Ref{Union{LoadingCache, Nothing}}(nothing) LoadingCache() = LoadingCache(load_path(), Dict(), Dict(), Dict(), Set(), Dict(), Dict(), Dict()) @@ -298,7 +298,7 @@ end ## package identification: determine unique identity of package to be loaded ## # Used by Pkg but not used in loading itself -function find_package(arg) +function find_package(arg) # ::Union{Nothing,String} pkgenv = identify_package_env(arg) pkgenv === nothing && return nothing pkg, env = pkgenv @@ -307,21 +307,21 @@ end """ Base.identify_package_env(name::String)::Union{Tuple{PkgId, String}, Nothing} - Base.identify_package_env(where::Union{Module,PkgId}, name::String)::Union{Tuple{PkgId, String} Nothing} + Base.identify_package_env(where::Union{Module,PkgId}, name::String)::Union{Tuple{PkgId, Union{String, Nothing}}, Nothing} Same as [`Base.identify_package`](@ref) except that the path to the environment where the package is identified -is also returned. +is also returned, except when the identity is not identified. """ identify_package_env(where::Module, name::String) = identify_package_env(PkgId(where), name) function identify_package_env(where::PkgId, name::String) cache = LOADING_CACHE[] if cache !== nothing - pkg_env = get(cache.identified_where, (where, name), nothing) - pkg_env === nothing || return pkg_env + pkg_env = get(cache.identified_where, (where, name), missing) + pkg_env === missing || return pkg_env end pkg_env = nothing if where.name === name - pkg_env = where, nothing + return (where, nothing) elseif where.uuid === nothing pkg_env = identify_package_env(name) # ignore `where` else @@ -342,8 +342,8 @@ end function identify_package_env(name::String) cache = LOADING_CACHE[] if cache !== nothing - pkg_env = get(cache.identified, name, nothing) - pkg_env === nothing || return pkg_env + pkg_env = get(cache.identified, name, missing) + pkg_env === missing || return pkg_env end pkg_env = nothing for env in load_path() @@ -390,17 +390,16 @@ identify_package(where::Module, name::String) = _nothing_or_first(identify_packa identify_package(where::PkgId, name::String) = _nothing_or_first(identify_package_env(where, name)) identify_package(name::String) = _nothing_or_first(identify_package_env(name)) -function locate_package_env(pkg::PkgId, stopenv::Union{String, Nothing}=nothing) +function locate_package_env(pkg::PkgId, stopenv::Union{String, Nothing}=nothing)::Union{Nothing,Tuple{String,String}} cache = LOADING_CACHE[] if cache !== nothing - pathenv = get(cache.located, (pkg, stopenv), nothing) - pathenv === nothing || return pathenv + pathenv = get(cache.located, (pkg, stopenv), missing) + pathenv === missing || return pathenv end path = nothing env′ = nothing if pkg.uuid === nothing for env in load_path() - env′ = env # look for the toplevel pkg `pkg.name` in this entry found = project_deps_get(env, pkg.name) if found !== nothing @@ -410,6 +409,7 @@ function locate_package_env(pkg::PkgId, stopenv::Union{String, Nothing}=nothing) # return the path the entry point for the code, if it could be found # otherwise, signal failure path = implicit_manifest_uuid_path(env, pkg) + env′ = env @goto done end end @@ -419,7 +419,6 @@ function locate_package_env(pkg::PkgId, stopenv::Union{String, Nothing}=nothing) end else for env in load_path() - env′ = env path = manifest_uuid_path(env, pkg) # missing is used as a sentinel to stop looking further down in envs if path === missing @@ -428,6 +427,7 @@ function locate_package_env(pkg::PkgId, stopenv::Union{String, Nothing}=nothing) end if path !== nothing path = entry_path(path, pkg.name) + env′ = env @goto done end if !(loading_extension || precompiling_extension) @@ -439,14 +439,16 @@ function locate_package_env(pkg::PkgId, stopenv::Union{String, Nothing}=nothing) mbypath = manifest_uuid_path(Sys.STDLIB, pkg) if mbypath isa String path = entry_path(mbypath, pkg.name) + env′ = Sys.STDLIB @goto done end end @label done if cache !== nothing - cache.located[(pkg, stopenv)] = path, env′ + cache.located[(pkg, stopenv)] = path === nothing ? nothing : (path, something(env′)) end - return path, env′ + path === nothing && return nothing + return path, something(env′) end """ @@ -1083,11 +1085,12 @@ end # want it to pick up caches that already exist for other optimization levels const ignore_compiled_cache = PkgId[] -function find_all_in_cache_path(pkg::PkgId) +function find_all_in_cache_path(pkg::PkgId, DEPOT_PATH::typeof(DEPOT_PATH)=DEPOT_PATH) paths = String[] pkg in ignore_compiled_cache && return paths entrypath, entryfile = cache_file_entry(pkg) - for path in joinpath.(DEPOT_PATH, entrypath) + for path in DEPOT_PATH + path = joinpath(path, entrypath) isdir(path) || continue for file in readdir(path, sort = false) # no sort given we sort later if !((pkg.uuid === nothing && file == entryfile * ".ji") || @@ -1142,6 +1145,8 @@ cachefile_from_ocachefile(cachefile) = string(chopsuffix(cachefile, ".$(Libc.Lib # use an Int counter so that nested @time_imports calls all remain open const TIMING_IMPORTS = Threads.Atomic{Int}(0) +# loads a precompile cache file, ignoring stale_cachefile tests +# assuming all depmods are already loaded and everything is valid # these return either the array of modules loaded from the path / content given # or an Exception that describes why it couldn't be loaded # and it reconnects the Base.Docs.META @@ -1169,6 +1174,15 @@ function _include_from_serialized(pkg::PkgId, path::String, ocachepath::Union{No t_comp_before = cumulative_compile_time_ns() end + for i in 1:length(depmods) + dep = depmods[i] + dep isa Module && continue + _, depkey, depbuild_id = dep::Tuple{String, PkgId, UInt128} + @assert root_module_exists(depkey) + dep = root_module(depkey) + depmods[i] = dep + end + if ocachepath !== nothing @debug "Loading object cache file $ocachepath for $(repr("text/plain", pkg))" sv = ccall(:jl_restore_package_image_from_file, Any, (Cstring, Any, Cint, Cstring, Cint), ocachepath, depmods, false, pkg.name, ignore_native) @@ -1314,9 +1328,6 @@ function insert_extension_triggers(pkg::PkgId) path_env_loc = locate_package_env(pkg) path_env_loc === nothing && return path, env_loc = path_env_loc - if path === nothing || env_loc === nothing - return - end insert_extension_triggers(env_loc, pkg) end @@ -1637,7 +1648,7 @@ function isprecompiled(pkg::PkgId; cachepaths::Vector{String}=Base.find_all_in_cache_path(pkg), sourcepath::Union{String,Nothing}=Base.locate_package(pkg), flags::CacheFlags=CacheFlags()) - isnothing(sourcepath) && error("Cannot locate source for $(repr(pkg))") + isnothing(sourcepath) && error("Cannot locate source for $(repr("text/plain", pkg))") for path_to_try in cachepaths staledeps = stale_cachefile(sourcepath, path_to_try, ignore_loaded = true, requested_flags=flags) if staledeps === true @@ -1674,10 +1685,9 @@ function isprecompiled(pkg::PkgId; return false end -# loads a precompile cache file, after checking stale_cachefile tests +# search for a precompile cache file to load, after some various checks function _tryrequire_from_serialized(modkey::PkgId, build_id::UInt128) assert_havelock(require_lock) - loaded = nothing if root_module_exists(modkey) loaded = root_module(modkey) else @@ -1687,7 +1697,7 @@ function _tryrequire_from_serialized(modkey::PkgId, build_id::UInt128) modpath = locate_package(modkey) modpath === nothing && return nothing set_pkgorigin_version_path(modkey, String(modpath)) - loaded = _require_search_from_serialized(modkey, String(modpath), build_id) + loaded = _require_search_from_serialized(modkey, String(modpath), build_id, true) finally end_loading(modkey, loaded) end @@ -1697,46 +1707,10 @@ function _tryrequire_from_serialized(modkey::PkgId, build_id::UInt128) end end end - if !(loaded isa Module) || PkgId(loaded) != modkey - return ErrorException("Required dependency $modkey failed to load from a cache file.") + if loaded isa Module && PkgId(loaded) == modkey && module_build_id(loaded) === build_id + return loaded end - return loaded -end - -# loads a precompile cache file, ignoring stale_cachefile tests -# assuming all depmods are already loaded and everything is valid -function _tryrequire_from_serialized(modkey::PkgId, path::String, ocachepath::Union{Nothing, String}, sourcepath::String, depmods::Vector{Any}) - assert_havelock(require_lock) - loaded = nothing - if root_module_exists(modkey) - loaded = root_module(modkey) - else - loaded = start_loading(modkey) - if loaded === nothing - try - for i in 1:length(depmods) - dep = depmods[i] - dep isa Module && continue - _, depkey, depbuild_id = dep::Tuple{String, PkgId, UInt128} - @assert root_module_exists(depkey) - dep = root_module(depkey) - depmods[i] = dep - end - set_pkgorigin_version_path(modkey, sourcepath) - loaded = _include_from_serialized(modkey, path, ocachepath, depmods) - finally - end_loading(modkey, loaded) - end - if loaded isa Module - insert_extension_triggers(modkey) - run_package_callbacks(modkey) - end - end - end - if !(loaded isa Module) || PkgId(loaded) != modkey - return ErrorException("Required dependency $modkey failed to load from a cache file.") - end - return loaded + return ErrorException("Required dependency $modkey failed to load from a cache file.") end # returns whether the package is tracked in coverage or malloc tracking based on @@ -1766,7 +1740,7 @@ function pkg_tracked(includes) end # loads a precompile cache file, ignoring stale_cachefile tests -# load the best available (non-stale) version of all dependent modules first +# load all dependent modules first function _tryrequire_from_serialized(pkg::PkgId, path::String, ocachepath::Union{Nothing, String}) assert_havelock(require_lock) local depmodnames @@ -1808,56 +1782,92 @@ end # returns `nothing` if require found a precompile cache for this sourcepath, but couldn't load it # returns the set of modules restored if the cache load succeeded -@constprop :none function _require_search_from_serialized(pkg::PkgId, sourcepath::String, build_id::UInt128; reasons=nothing) +@constprop :none function _require_search_from_serialized(pkg::PkgId, sourcepath::String, build_id::UInt128, stalecheck::Bool; reasons=nothing, DEPOT_PATH::typeof(DEPOT_PATH)=DEPOT_PATH) assert_havelock(require_lock) - paths = find_all_in_cache_path(pkg) + paths = find_all_in_cache_path(pkg, DEPOT_PATH) + newdeps = PkgId[] for path_to_try in paths::Vector{String} - staledeps = stale_cachefile(pkg, build_id, sourcepath, path_to_try; reasons) + staledeps = stale_cachefile(pkg, build_id, sourcepath, path_to_try; reasons, stalecheck) if staledeps === true continue end - staledeps, ocachefile = staledeps::Tuple{Vector{Any}, Union{Nothing, String}} - # finish checking staledeps module graph - for i in 1:length(staledeps) - dep = staledeps[i] - dep isa Module && continue - modpath, modkey, modbuild_id = dep::Tuple{String, PkgId, UInt128} - modpaths = find_all_in_cache_path(modkey) - for modpath_to_try in modpaths - modstaledeps = stale_cachefile(modkey, modbuild_id, modpath, modpath_to_try) - if modstaledeps === true - continue - end - modstaledeps, modocachepath = modstaledeps::Tuple{Vector{Any}, Union{Nothing, String}} - staledeps[i] = (modpath, modkey, modpath_to_try, modstaledeps, modocachepath) - @goto check_next_dep - end - @debug "Rejecting cache file $path_to_try because required dependency $modkey with build ID $(UUID(modbuild_id)) is missing from the cache." - @goto check_next_path - @label check_next_dep - end try - touch(path_to_try) # update timestamp of precompilation file - catch ex # file might be read-only and then we fail to update timestamp, which is fine - ex isa IOError || rethrow() - end - # finish loading module graph into staledeps - for i in 1:length(staledeps) - dep = staledeps[i] - dep isa Module && continue - modpath, modkey, modcachepath, modstaledeps, modocachepath = dep::Tuple{String, PkgId, String, Vector{Any}, Union{Nothing, String}} - dep = _tryrequire_from_serialized(modkey, modcachepath, modocachepath, modpath, modstaledeps) - if !isa(dep, Module) - @debug "Rejecting cache file $path_to_try because required dependency $modkey failed to load from cache file for $modcachepath." exception=dep + staledeps, ocachefile = staledeps::Tuple{Vector{Any}, Union{Nothing, String}} + # finish checking staledeps module graph + for i in 1:length(staledeps) + dep = staledeps[i] + dep isa Module && continue + modpath, modkey, modbuild_id = dep::Tuple{String, PkgId, UInt128} + modpaths = find_all_in_cache_path(modkey, DEPOT_PATH) + for modpath_to_try in modpaths + modstaledeps = stale_cachefile(modkey, modbuild_id, modpath, modpath_to_try; stalecheck) + if modstaledeps === true + continue + end + modstaledeps, modocachepath = modstaledeps::Tuple{Vector{Any}, Union{Nothing, String}} + staledeps[i] = (modpath, modkey, modbuild_id, modpath_to_try, modstaledeps, modocachepath) + @goto check_next_dep + end + @debug "Rejecting cache file $path_to_try because required dependency $modkey with build ID $(UUID(modbuild_id)) is missing from the cache." @goto check_next_path + @label check_next_dep + end + if stalecheck + try + touch(path_to_try) # update timestamp of precompilation file + catch ex # file might be read-only and then we fail to update timestamp, which is fine + ex isa IOError || rethrow() + end end - staledeps[i] = dep + # finish loading module graph into staledeps + for i in 1:length(staledeps) + dep = staledeps[i] + dep isa Module && continue + modpath, modkey, modbuild_id, modcachepath, modstaledeps, modocachepath = dep::Tuple{String, PkgId, UInt128, String, Vector{Any}, Union{Nothing, String}} + dep = nothing + if root_module_exists(modkey) + dep = root_module(modkey) + end + while true + if dep isa Module + if PkgId(dep) == modkey && module_build_id(dep) === modbuild_id + break + else + if stalecheck + @debug "Rejecting cache file $path_to_try because module $modkey is already loaded and incompatible." + @goto check_next_path + end + end + end + dep = start_loading(modkey) + if dep === nothing + try + set_pkgorigin_version_path(modkey, modpath) + dep = _include_from_serialized(modkey, modcachepath, modocachepath, modstaledeps) + finally + end_loading(modkey, dep) + end + if !isa(dep, Module) + @debug "Rejecting cache file $path_to_try because required dependency $modkey failed to load from cache file for $modcachepath." exception=dep + @goto check_next_path + else + push!(newdeps, modkey) + end + end + end + staledeps[i] = dep + end + restored = _include_from_serialized(pkg, path_to_try, ocachefile, staledeps) + isa(restored, Module) && return restored + @debug "Deserialization checks failed while attempting to load cache from $path_to_try" exception=restored + @label check_next_path + finally + for modkey in newdeps + insert_extension_triggers(modkey) + run_package_callbacks(modkey) + end + empty!(newdeps) end - restored = _include_from_serialized(pkg, path_to_try, ocachefile, staledeps) - isa(restored, Module) && return restored - @debug "Deserialization checks failed while attempting to load cache from $path_to_try" exception=restored - continue - @label check_next_path end return nothing end @@ -2241,14 +2251,12 @@ function unreference_module(key::PkgId) end # whoever takes the package_locks[pkg] must call this function immediately -function set_pkgorigin_version_path(pkg::PkgId, path::Union{String,Nothing}) +function set_pkgorigin_version_path(pkg::PkgId, path::String) assert_havelock(require_lock) pkgorigin = get!(PkgOrigin, pkgorigins, pkg) - if path !== nothing - # Pkg needs access to the version of packages in the sysimage. - if generating_output(#=incremental=#false) - pkgorigin.version = get_pkgversion_from_path(joinpath(dirname(path), "..")) - end + # Pkg needs access to the version of packages in the sysimage. + if generating_output(#=incremental=#false) + pkgorigin.version = get_pkgversion_from_path(joinpath(dirname(path), "..")) end pkgorigin.path = path nothing @@ -2281,7 +2289,7 @@ function _require(pkg::PkgId, env=nothing) # attempt to load the module file via the precompile cache locations if JLOptions().use_compiled_modules != 0 @label load_from_cache - m = _require_search_from_serialized(pkg, path, UInt128(0); reasons) + m = _require_search_from_serialized(pkg, path, UInt128(0), true; reasons) if m isa Module return m end @@ -2319,7 +2327,7 @@ function _require(pkg::PkgId, env=nothing) # spawn off a new incremental pre-compile task for recursive `require` calls cachefile_or_module = maybe_cachefile_lock(pkg, path) do # double-check now that we have lock - m = _require_search_from_serialized(pkg, path, UInt128(0)) + m = _require_search_from_serialized(pkg, path, UInt128(0), true) m isa Module && return m compilecache(pkg, path; reasons) end @@ -2376,10 +2384,10 @@ function _require(pkg::PkgId, env=nothing) return loaded end -# Only used from test/precompile.jl -function _require_from_serialized(uuidkey::PkgId, path::String, ocachepath::Union{String, Nothing}) +# load a serialized file directly +function _require_from_serialized(uuidkey::PkgId, path::String, ocachepath::Union{String, Nothing}, sourcepath::String) @lock require_lock begin - set_pkgorigin_version_path(uuidkey, nothing) + set_pkgorigin_version_path(uuidkey, sourcepath) newm = _tryrequire_from_serialized(uuidkey, path, ocachepath) newm isa Module || throw(newm) insert_extension_triggers(uuidkey) @@ -2389,6 +2397,58 @@ function _require_from_serialized(uuidkey::PkgId, path::String, ocachepath::Unio end end +# load a serialized file directly from append_bundled_depot_path for uuidkey without stalechecks +function require_stdlib(uuidkey::PkgId, ext::Union{Nothing, String}=nothing) + @lock require_lock begin + if root_module_exists(uuidkey) + return loaded_modules[uuidkey] + end + # first since this is a stdlib, try to look there directly first + env = Sys.STDLIB + #sourcepath = "" + if ext === nothing + sourcepath = normpath(env, uuidkey.name, "src", uuidkey.name * ".jl") + else + sourcepath = find_ext_path(normpath(joinpath(env, uuidkey.name)), ext) + uuidkey = PkgId(uuid5(uuidkey.uuid, ext), ext) + end + #mbypath = manifest_uuid_path(env, uuidkey) + #if mbypath isa String + # sourcepath = entry_path(mbypath, uuidkey.name) + #else + # # if the user deleted the stdlib folder, we next try using their environment + # sourcepath = locate_package_env(uuidkey) + # if sourcepath !== nothing + # sourcepath, env = sourcepath + # end + #end + #if sourcepath === nothing + # throw(ArgumentError(""" + # Package $(repr("text/plain", uuidkey)) is required but does not seem to be installed. + # """)) + #end + set_pkgorigin_version_path(uuidkey, sourcepath) + depot_path = append_bundled_depot_path!(empty(DEPOT_PATH)) + newm = start_loading(uuidkey) + newm === nothing || return newm + try + newm = _require_search_from_serialized(uuidkey, sourcepath, UInt128(0), false; DEPOT_PATH=depot_path) + finally + end_loading(uuidkey, newm) + end + if newm isa Module + # After successfully loading, notify downstream consumers + insert_extension_triggers(env, uuidkey) + run_package_callbacks(uuidkey) + else + # if the user deleted their bundled depot, next try to load it completely normally + newm = _require(uuidkey) + end + return newm + end +end + + # relative-path load @@ -3391,7 +3451,9 @@ list_reasons(::Nothing) = "" return stale_cachefile(PkgId(""), UInt128(0), modpath, cachefile; ignore_loaded, requested_flags, reasons) end @constprop :none function stale_cachefile(modkey::PkgId, build_id::UInt128, modpath::String, cachefile::String; - ignore_loaded::Bool = false, requested_flags::CacheFlags=CacheFlags(), reasons::Union{Dict{String,Int},Nothing}=nothing) + ignore_loaded::Bool=false, requested_flags::CacheFlags=CacheFlags(), + reasons::Union{Dict{String,Int},Nothing}=nothing, stalecheck::Bool=true) + # XXX: this function appears to dl all of the file validation, not just those checks related to stale io = open(cachefile, "r") try checksum = isvalid_cache_header(io) @@ -3466,7 +3528,7 @@ end M = root_module(req_key) if PkgId(M) == req_key && module_build_id(M) === req_build_id depmods[i] = M - elseif ignore_loaded + elseif ignore_loaded || !stalecheck # Used by Pkg.precompile given that there it's ok to precompile different versions of loaded packages @goto locate_branch else @@ -3476,7 +3538,7 @@ end end else @label locate_branch - path = locate_package(req_key) + path = locate_package(req_key) # TODO: add env and/or skip this when stalecheck is false if path === nothing @debug "Rejecting cache file $cachefile because dependency $req_key not found." record_reason(reasons, "dep missing source") @@ -3489,13 +3551,12 @@ end # check if this file is going to provide one of our concrete dependencies # or if it provides a version that conflicts with our concrete dependencies # or neither - skip_check = false for (req_key, req_build_id) in _concrete_dependencies build_id = get(modules, req_key, UInt64(0)) if build_id !== UInt64(0) build_id |= UInt128(checksum) << 64 if build_id === req_build_id - skip_check = true + stalecheck = false break end @debug "Rejecting cache file $cachefile because it provides the wrong build_id (got $((UUID(build_id)))) for $req_key (want $(UUID(req_build_id)))" @@ -3505,7 +3566,7 @@ end end # now check if this file's content hash has changed relative to its source files - if !skip_check + if stalecheck if !samefile(includes[1].filename, modpath) && !samefile(fixup_stdlib_path(includes[1].filename), modpath) @debug "Rejecting cache file $cachefile because it is for file $(includes[1].filename) not file $modpath" record_reason(reasons, "wrong source") diff --git a/base/stream.jl b/base/stream.jl index 3de61181e978d..3264b8f153677 100644 --- a/base/stream.jl +++ b/base/stream.jl @@ -304,7 +304,7 @@ function init_stdio(handle::Ptr{Cvoid}) elseif t == UV_TTY io = TTY(handle, StatusOpen) elseif t == UV_TCP - Sockets = require(PkgId(UUID((0x6462fe0b_24de_5631, 0x8697_dd941f90decc)), "Sockets")) + Sockets = require_stdlib(PkgId(UUID((0x6462fe0b_24de_5631, 0x8697_dd941f90decc)), "Sockets")) io = Sockets.TCPSocket(handle, StatusOpen) elseif t == UV_NAMED_PIPE io = PipeEndpoint(handle, StatusOpen) @@ -341,7 +341,7 @@ function open(h::OS_HANDLE) elseif t == UV_TTY io = TTY(h) elseif t == UV_TCP - Sockets = require(PkgId(UUID((0x6462fe0b_24de_5631, 0x8697_dd941f90decc)), "Sockets")) + Sockets = require_stdlib(PkgId(UUID((0x6462fe0b_24de_5631, 0x8697_dd941f90decc)), "Sockets")) io = Sockets.TCPSocket(h) elseif t == UV_NAMED_PIPE io = PipeEndpoint(h) diff --git a/base/util.jl b/base/util.jl index a26a874c7a461..847d89e895af4 100644 --- a/base/util.jl +++ b/base/util.jl @@ -695,7 +695,7 @@ function runtests(tests = ["all"]; ncores::Int = ceil(Int, Sys.CPU_THREADS / 2), catch buf = PipeBuffer() original_load_path = copy(Base.LOAD_PATH); empty!(Base.LOAD_PATH); pushfirst!(Base.LOAD_PATH, "@stdlib") - let InteractiveUtils = Base.require(Base, :InteractiveUtils) + let InteractiveUtils = Base.require_stdlib(Base, :InteractiveUtils) @invokelatest InteractiveUtils.versioninfo(buf) end empty!(Base.LOAD_PATH); append!(Base.LOAD_PATH, original_load_path) diff --git a/stdlib/InteractiveUtils/src/InteractiveUtils.jl b/stdlib/InteractiveUtils/src/InteractiveUtils.jl index 7391a0d6b2f00..519acfc8a2f33 100644 --- a/stdlib/InteractiveUtils/src/InteractiveUtils.jl +++ b/stdlib/InteractiveUtils/src/InteractiveUtils.jl @@ -338,7 +338,7 @@ export peakflops function peakflops(n::Integer=4096; eltype::DataType=Float64, ntrials::Integer=3, parallel::Bool=false) # Base.depwarn("`peakflops` has moved to the LinearAlgebra module, " * # "add `using LinearAlgebra` to your imports.", :peakflops) - let LinearAlgebra = Base.require(Base.PkgId( + let LinearAlgebra = Base.require_stdlib(Base.PkgId( Base.UUID((0x37e2e46d_f89d_539d,0xb4ee_838fcccc9c8e)), "LinearAlgebra")) return LinearAlgebra.peakflops(n, eltype=eltype, ntrials=ntrials, parallel=parallel) end @@ -353,7 +353,7 @@ function report_bug(kind) if Base.locate_package(BugReportingId) === nothing @info "Package `BugReporting` not found - attempting temporary installation" # Create a temporary environment and add BugReporting - let Pkg = Base.require(Base.PkgId( + let Pkg = Base.require_stdlib(Base.PkgId( Base.UUID((0x44cfe95a_1eb2_52ea,0xb672_e2afdf69b78f)), "Pkg")) mktempdir() do tmp old_load_path = copy(LOAD_PATH) diff --git a/stdlib/LinearAlgebra/src/LinearAlgebra.jl b/stdlib/LinearAlgebra/src/LinearAlgebra.jl index 793775992fdea..acacf4b20cfc7 100644 --- a/stdlib/LinearAlgebra/src/LinearAlgebra.jl +++ b/stdlib/LinearAlgebra/src/LinearAlgebra.jl @@ -691,7 +691,7 @@ function peakflops(n::Integer=4096; eltype::DataType=Float64, ntrials::Integer=3 end if parallel - let Distributed = Base.require(Base.PkgId( + let Distributed = Base.require_stdlib(Base.PkgId( Base.UUID((0x8ba89e20_285c_5b6f, 0x9357_94700520ee1b)), "Distributed")) nworkers = @invokelatest Distributed.nworkers() results = @invokelatest Distributed.pmap(peakflops, fill(n, nworkers)) diff --git a/stdlib/REPL/src/REPL.jl b/stdlib/REPL/src/REPL.jl index f39aed51cebaf..840e6daaf1909 100644 --- a/stdlib/REPL/src/REPL.jl +++ b/stdlib/REPL/src/REPL.jl @@ -1188,25 +1188,23 @@ function setup_interface( ']' => function (s::MIState,o...) if isempty(s) || position(LineEdit.buffer(s)) == 0 pkgid = Base.PkgId(Base.UUID("44cfe95a-1eb2-52ea-b672-e2afdf69b78f"), "Pkg") - if Base.locate_package(pkgid) !== nothing # Only try load Pkg if we can find it - Pkg = Base.require(pkgid) - REPLExt = Base.get_extension(Pkg, :REPLExt) - # Pkg should have loaded its REPL mode by now, let's find it so we can transition to it. - pkg_mode = nothing + REPLExt = Base.require_stdlib(pkgid, "REPLExt") + pkg_mode = nothing + if REPLExt isa Module && isdefined(REPLExt, :PkgCompletionProvider) for mode in repl.interface.modes if mode isa LineEdit.Prompt && mode.complete isa REPLExt.PkgCompletionProvider pkg_mode = mode break end end - # TODO: Cache the `pkg_mode`? - if pkg_mode !== nothing - buf = copy(LineEdit.buffer(s)) - transition(s, pkg_mode) do - LineEdit.state(s, pkg_mode).input_buffer = buf - end - return + end + # TODO: Cache the `pkg_mode`? + if pkg_mode !== nothing + buf = copy(LineEdit.buffer(s)) + transition(s, pkg_mode) do + LineEdit.state(s, pkg_mode).input_buffer = buf end + return end end edit_insert(s, ']') diff --git a/test/precompile.jl b/test/precompile.jl index a68d8936d1ed1..344b98b2ee186 100644 --- a/test/precompile.jl +++ b/test/precompile.jl @@ -407,11 +407,11 @@ precompile_test_harness(false) do dir else ocachefile = nothing end - # use _require_from_serialized to ensure that the test fails if - # the module doesn't reload from the image: + # use _require_from_serialized to ensure that the test fails if + # the module doesn't reload from the image: @test_warn "@ccallable was already defined for this method name" begin @test_logs (:warn, "Replacing module `$Foo_module`") begin - m = Base._require_from_serialized(Base.PkgId(Foo), cachefile, ocachefile) + m = Base._require_from_serialized(Base.PkgId(Foo), cachefile, ocachefile, Foo_file) @test isa(m, Module) end end From 945c93fb1b62c21b99ffcf66b3b8aab295e021b8 Mon Sep 17 00:00:00 2001 From: Florian Date: Tue, 20 Feb 2024 17:59:29 +0100 Subject: [PATCH 19/25] Add `Sys.isreadable, Sys.iswriteable`, update `ispath` (#53320) As discussed here: https://github.com/JuliaLang/julia/pull/53286#discussion_r1487039190 Readds the methods that were removed in https://github.com/JuliaLang/julia/pull/12819. (cherry picked from commit ea2b255b3e42fc9abee3a657f5a3f785beb9edff) --- base/stat.jl | 10 +++++++ base/sysinfo.jl | 65 +++++++++++++++++++++++++++++++++++++++++++- doc/src/base/base.md | 2 ++ test/file.jl | 31 +++++++++++++++++---- 4 files changed, 102 insertions(+), 6 deletions(-) diff --git a/base/stat.jl b/base/stat.jl index 56b960c4f74ea..3931b9d1f3583 100644 --- a/base/stat.jl +++ b/base/stat.jl @@ -326,6 +326,16 @@ otherwise returns `false`. This is the generalization of [`isfile`](@ref), [`isdir`](@ref) etc. """ ispath(st::StatStruct) = filemode(st) & 0xf000 != 0x0000 +function ispath(path::String) + # We use `access()` and `F_OK` to determine if a given path exists. `F_OK` comes from `unistd.h`. + F_OK = 0x00 + r = ccall(:jl_fs_access, Cint, (Cstring, Cint), path, F_OK) + if !(r in (0, Base.UV_ENOENT, Base.UV_ENOTDIR, Base.UV_EINVAL)) + uv_error(string("ispath(", repr(path), ")"), r) + end + return r == 0 +end +ispath(path::AbstractString) = ispath(String(path)) """ isfifo(path) -> Bool diff --git a/base/sysinfo.jl b/base/sysinfo.jl index 97e5e2a71bcbc..c5744873312d6 100644 --- a/base/sysinfo.jl +++ b/base/sysinfo.jl @@ -33,6 +33,8 @@ export BINDIR, iswindows, isjsvm, isexecutable, + isreadable, + iswriteable, username, which @@ -556,20 +558,81 @@ const WINDOWS_VISTA_VER = v"6.0" Return `true` if the given `path` has executable permissions. +!!! note + This permission may change before the user executes `path`, + so it is recommended to execute the file and handle the error if that fails, + rather than calling `isexecutable` first. + !!! note Prior to Julia 1.6, this did not correctly interrogate filesystem ACLs on Windows, therefore it would return `true` for any file. From Julia 1.6 on, it correctly determines whether the file is marked as executable or not. + +See also [`ispath`](@ref), [`isreadable`](@ref), [`iswriteable`](@ref). """ function isexecutable(path::String) # We use `access()` and `X_OK` to determine if a given path is # executable by the current user. `X_OK` comes from `unistd.h`. X_OK = 0x01 - return ccall(:jl_fs_access, Cint, (Ptr{UInt8}, Cint), path, X_OK) == 0 + return ccall(:jl_fs_access, Cint, (Cstring, Cint), path, X_OK) == 0 end isexecutable(path::AbstractString) = isexecutable(String(path)) +""" + Sys.isreadable(path::String) + +Return `true` if the access permissions for the given `path` permitted reading by the current user. + +!!! note + This permission may change before the user calls `open`, + so it is recommended to just call `open` alone and handle the error if that fails, + rather than calling `isreadable` first. + +!!! note + Currently this function does not correctly interrogate filesystem + ACLs on Windows, therefore it can return wrong results. + +!!! compat "Julia 1.11" + This function requires at least Julia 1.11. + +See also [`ispath`](@ref), [`isexecutable`](@ref), [`iswriteable`](@ref). +""" +function isreadable(path::String) + # We use `access()` and `R_OK` to determine if a given path is + # readable by the current user. `R_OK` comes from `unistd.h`. + R_OK = 0x04 + return ccall(:jl_fs_access, Cint, (Cstring, Cint), path, R_OK) == 0 +end +isreadable(path::AbstractString) = isreadable(String(path)) + +""" + Sys.iswriteable(path::String) + +Return `true` if the access permissions for the given `path` permitted writing by the current user. + +!!! note + This permission may change before the user calls `open`, + so it is recommended to just call `open` alone and handle the error if that fails, + rather than calling `iswriteable` first. + +!!! note + Currently this function does not correctly interrogate filesystem + ACLs on Windows, therefore it can return wrong results. + +!!! compat "Julia 1.11" + This function requires at least Julia 1.11. + +See also [`ispath`](@ref), [`isexecutable`](@ref), [`isreadable`](@ref). +""" +function iswriteable(path::String) + # We use `access()` and `W_OK` to determine if a given path is + # writeable by the current user. `W_OK` comes from `unistd.h`. + W_OK = 0x02 + return ccall(:jl_fs_access, Cint, (Cstring, Cint), path, W_OK) == 0 +end +iswriteable(path::AbstractString) = iswriteable(String(path)) + """ Sys.which(program_name::String) diff --git a/doc/src/base/base.md b/doc/src/base/base.md index 3fd2bbb12bb1f..556d51d3af0c1 100644 --- a/doc/src/base/base.md +++ b/doc/src/base/base.md @@ -374,6 +374,8 @@ Base.Sys.uptime Base.Sys.isjsvm Base.Sys.loadavg Base.Sys.isexecutable +Base.Sys.isreadable +Base.Sys.iswriteable Base.Sys.username Base.@static ``` diff --git a/test/file.jl b/test/file.jl index 8f62b533083c6..c32f655eb677e 100644 --- a/test/file.jl +++ b/test/file.jl @@ -1660,23 +1660,44 @@ else ) end -@testset "chmod/isexecutable" begin +@testset "chmod/isexecutable/isreadable/iswriteable" begin mktempdir() do dir - mkdir(joinpath(dir, "subdir")) + subdir = joinpath(dir, "subdir") fpath = joinpath(dir, "subdir", "foo") - # Test that we can actually set the executable bit on all platforms. + @test !ispath(subdir) + mkdir(subdir) + @test ispath(subdir) + + @test !ispath(fpath) touch(fpath) + @test ispath(fpath) + + # Test that we can actually set the executable/readable/writeable bit on all platforms. chmod(fpath, 0o644) @test !Sys.isexecutable(fpath) + @test Sys.isreadable(fpath) + Sys.iswindows() ? @test_skip(Sys.iswriteable(fpath)) : @test(Sys.iswriteable(fpath)) chmod(fpath, 0o755) @test Sys.isexecutable(fpath) + @test Sys.isreadable(fpath) + Sys.iswindows() ? @test_skip(Sys.iswriteable(fpath)) : @test(Sys.iswriteable(fpath)) + chmod(fpath, 0o444) + @test !Sys.isexecutable(fpath) + @test Sys.isreadable(fpath) + @test !Sys.iswriteable(fpath) + chmod(fpath, 0o244) + @test !Sys.isexecutable(fpath) + Sys.iswindows() ? @test_skip(!Sys.isreadable(fpath)) : @test(!Sys.isreadable(fpath)) + Sys.iswindows() ? @test_skip(Sys.iswriteable(fpath)) : @test(Sys.iswriteable(fpath)) # Ensure that, on Windows, where inheritance is default, # chmod still behaves as we expect. if Sys.iswindows() - chmod(joinpath(dir, "subdir"), 0o666) - @test Sys.isexecutable(fpath) + chmod(subdir, 0o666) + @test !Sys.isexecutable(fpath) + @test Sys.isreadable(fpath) + @test_skip Sys.iswriteable(fpath) end # Reset permissions to all at the end, so it can be deleted properly. From 4141f35ef4afa9583705bd756ad344fe0fffe808 Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Fri, 23 Feb 2024 11:16:17 -0500 Subject: [PATCH 20/25] staticdata: fix assert from partially disabled native code (#53439) This should fix the assertion failure that has been plaguing the Pkg tests, as discussed in https://github.com/JuliaLang/julia/pull/52123#issuecomment-1959965395 (cherry picked from commit 6cbed3198554381a6ed57b2d81f6c2ae0adfc4e5) --- src/staticdata.c | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/staticdata.c b/src/staticdata.c index f899899aff242..c8fd4053aaf73 100644 --- a/src/staticdata.c +++ b/src/staticdata.c @@ -3768,8 +3768,11 @@ JL_DLLEXPORT jl_value_t *jl_restore_package_image_from_file(const char *fname, j jl_image_t pkgimage = jl_init_processor_pkgimg(pkgimg_handle); - if (ignore_native){ - memset(&pkgimage.fptrs, 0, sizeof(pkgimage.fptrs)); + if (ignore_native) { + // Must disable using native code in possible downstream users of this code: + // https://github.com/JuliaLang/julia/pull/52123#issuecomment-1959965395. + // The easiest way to do that is to disable it in all of them. + jl_options.use_sysimage_native_code = JL_OPTIONS_USE_SYSIMAGE_NATIVE_CODE_NO; } jl_value_t* mod = jl_restore_incremental_from_buf(pkgimg_handle, pkgimg_data, &pkgimage, *plen, depmods, completeinfo, pkgname, 0); From 7a95a0cc8b578e347914367016e3e7e2ff5d5937 Mon Sep 17 00:00:00 2001 From: Valentin Churavy Date: Sat, 24 Feb 2024 21:48:37 -0500 Subject: [PATCH 21/25] Prevent tainting native code loading from propagating (#53457) When we use options like code coverage, we can't use the native code present in the cache file since it is not instrumented. PR #52123 introduced the capability of skipping the native code during loading, but created the issue that subsequent packages could have an explicit or implicit dependency on the native code. PR #53439 tainted the current process by setting `use_sysimage_native_code`, but this flag is propagated to subprocesses and lead to a regression in test time. Move this to a process local flag to avoid the regression. In the future we might be able to change the calling convention for cross-image calls to `invoke(ci::CodeInstance, args...)` instead of `ci.fptr(args...)` to handle native code not being present. --------- Co-authored-by: Jameson Nash (cherry picked from commit b8a0a3978ccf163ce8fc371a4882547dc1271d53) --- src/staticdata.c | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/staticdata.c b/src/staticdata.c index c8fd4053aaf73..261042b775c14 100644 --- a/src/staticdata.c +++ b/src/staticdata.c @@ -3066,6 +3066,11 @@ JL_DLLEXPORT void jl_set_sysimg_so(void *handle) extern void rebuild_image_blob_tree(void); extern void export_jl_small_typeof(void); +// When an image is loaded with ignore_native, all subsequent image loads must ignore +// native code in the cache-file since we can't gurantuee that there are no call edges +// into the native code of the image. See https://github.com/JuliaLang/julia/pull/52123#issuecomment-1959965395. +int IMAGE_NATIVE_CODE_TAINTED = 0; + static void jl_restore_system_image_from_stream_(ios_t *f, jl_image_t *image, jl_array_t *depmods, uint64_t checksum, /* outputs */ jl_array_t **restored, jl_array_t **init_order, jl_array_t **extext_methods, jl_array_t **internal_methods, @@ -3092,9 +3097,10 @@ static void jl_restore_system_image_from_stream_(ios_t *f, jl_image_t *image, jl // in --build mode only use sysimg data, not precompiled native code int imaging_mode = jl_generating_output() && !jl_options.incremental; - if (imaging_mode || jl_options.use_sysimage_native_code != JL_OPTIONS_USE_SYSIMAGE_NATIVE_CODE_YES) { + if (imaging_mode || jl_options.use_sysimage_native_code != JL_OPTIONS_USE_SYSIMAGE_NATIVE_CODE_YES || IMAGE_NATIVE_CODE_TAINTED) { memset(&image->fptrs, 0, sizeof(image->fptrs)); image->gvars_base = NULL; + IMAGE_NATIVE_CODE_TAINTED = 1; } // step 1: read section map @@ -3772,7 +3778,7 @@ JL_DLLEXPORT jl_value_t *jl_restore_package_image_from_file(const char *fname, j // Must disable using native code in possible downstream users of this code: // https://github.com/JuliaLang/julia/pull/52123#issuecomment-1959965395. // The easiest way to do that is to disable it in all of them. - jl_options.use_sysimage_native_code = JL_OPTIONS_USE_SYSIMAGE_NATIVE_CODE_NO; + IMAGE_NATIVE_CODE_TAINTED = 1; } jl_value_t* mod = jl_restore_incremental_from_buf(pkgimg_handle, pkgimg_data, &pkgimage, *plen, depmods, completeinfo, pkgname, 0); From ba189d84fa31387930a643f68f6cf4c564c25e84 Mon Sep 17 00:00:00 2001 From: Max Horn Date: Fri, 23 Feb 2024 09:19:02 +0100 Subject: [PATCH 22/25] Add update mechanism for Terminfo, and common user-alias data (#53285) Now that we take care of terminfo parsing ourselves, having a clear origin and processing method for arriving at our reference terminfo data seems somewhat important. The original form was acquired by re-purposing some pre-processed terminfo data from NCurses (I forget the exact source). I considered adding a separate ingestion/processing script, but it occurred to me that it would make sense to have the method for updating the data file be _in_ the data file, by turning it into a quine. This seems to work rather well, and so long as the NCurses source file format stays the same, updating the terminfo data is now dead simple. While working on the terminfo files, some minor refactors popped up as "probably nice to have". One of which makes the reported number of flags/numbers/strings actually accurate. Lastly, to support the ergonomic use of capability variable names instead of the short (read: uninformative) capname, we now also import the NCurses-recognised extended capabilities, and generate/specify some nice aliases for them. ----- If we separate out the terminfo parser/data into a small stdlib, the state here will be the initial state of the repo. (cherry picked from commit 923fe2d74d469d73f0285a9cd41c88f90edd04d1) --- base/terminfo.jl | 37 +- base/terminfo_data.jl | 1266 +++++++++++++++++++++++++---------------- test/terminfo.jl | 48 +- 3 files changed, 818 insertions(+), 533 deletions(-) diff --git a/base/terminfo.jl b/base/terminfo.jl index 3792e35e0fd8d..6f1d1ca8015f0 100644 --- a/base/terminfo.jl +++ b/base/terminfo.jl @@ -66,9 +66,10 @@ struct TermInfo numbers::Dict{Symbol, Int} strings::Dict{Symbol, String} extensions::Union{Nothing, Set{Symbol}} + aliases::Dict{Symbol, Symbol} end -TermInfo() = TermInfo([], Dict(), Dict(), Dict(), nothing) +TermInfo() = TermInfo([], Dict(), Dict(), Dict(), nothing, Dict()) function read(data::IO, ::Type{TermInfoRaw}) # Parse according to `term(5)` @@ -175,24 +176,28 @@ function TermInfo(raw::TermInfoRaw) flags = Dict{Symbol, Bool}() numbers = Dict{Symbol, Int}() strings = Dict{Symbol, String}() + aliases = Dict{Symbol, Symbol}() extensions = nothing for (flag, value) in zip(TERM_FLAGS, raw.flags) - flags[flag.short] = value - flags[flag.long] = value + flags[flag.name] = value + aliases[flag.capname] = flag.name end for (num, value) in zip(TERM_NUMBERS, raw.numbers) - numbers[num.short] = Int(value) - numbers[num.long] = Int(value) + numbers[num.name] = Int(value) + aliases[num.capname] = num.name end for (str, value) in zip(TERM_STRINGS, raw.strings) if !isnothing(value) - strings[str.short] = value - strings[str.long] = value + strings[str.name] = value + aliases[str.capname] = str.name end end if !isnothing(raw.extended) extensions = Set{Symbol}() - for (key, value) in raw.extended + longalias(key, value) = first(get(TERM_USER, (typeof(value), key), (nothing, ""))) + for (short, value) in raw.extended + long = longalias(short, value) + key = something(long, short) push!(extensions, key) if value isa Bool flags[key] = value @@ -201,26 +206,30 @@ function TermInfo(raw::TermInfoRaw) elseif value isa String strings[key] = value end + if !isnothing(long) + aliases[short] = long + end end end - TermInfo(raw.names, flags, numbers, strings, extensions) + TermInfo(raw.names, flags, numbers, strings, extensions, aliases) end -get(ti::TermInfo, key::Symbol, default::Bool) = get(ti.flags, key, default) -get(ti::TermInfo, key::Symbol, default::Int) = get(ti.numbers, key, default) -get(ti::TermInfo, key::Symbol, default::String) = get(ti.strings, key, default) +get(ti::TermInfo, key::Symbol, default::Bool) = get(ti.flags, get(ti.aliases, key, key), default) +get(ti::TermInfo, key::Symbol, default::Int) = get(ti.numbers, get(ti.aliases, key, key), default) +get(ti::TermInfo, key::Symbol, default::String) = get(ti.strings, get(ti.aliases, key, key), default) haskey(ti::TermInfo, key::Symbol) = - haskey(ti.flags, key) || haskey(ti.numbers, key) || haskey(ti.strings, key) + haskey(ti.flags, key) || haskey(ti.numbers, key) || haskey(ti.strings, key) || haskey(ti.aliases, key) function getindex(ti::TermInfo, key::Symbol) haskey(ti.flags, key) && return ti.flags[key] haskey(ti.numbers, key) && return ti.numbers[key] haskey(ti.strings, key) && return ti.strings[key] + haskey(ti.aliases, key) && return getindex(ti, ti.aliases[key]) throw(KeyError(key)) end -keys(ti::TermInfo) = keys(ti.flags) ∪ keys(ti.numbers) ∪ keys(ti.strings) +keys(ti::TermInfo) = keys(ti.flags) ∪ keys(ti.numbers) ∪ keys(ti.strings) ∪ keys(ti.aliases) function show(io::IO, ::MIME"text/plain", ti::TermInfo) print(io, "TermInfo(", ti.names, "; ", length(ti.flags), " flags, ", diff --git a/base/terminfo_data.jl b/base/terminfo_data.jl index 38c058f414f07..caf2ff528d3e1 100644 --- a/base/terminfo_data.jl +++ b/base/terminfo_data.jl @@ -1,5 +1,15 @@ # This file is a part of Julia. License is MIT: https://julialang.org/license +# Updating this listing is fairly easy, assuming existence of a unix system, +# posix shell, and `awk`. Just update the version string in the commented out +# `NCURSES_VERSION` variable, and run this file. This works because this file is +# a bit of a quine. + +#= +awk '/^#=run/{flag=1;next}/=#/{flag=0}flag{gsub(/__FILE__/,"\"'"$0"'\"");print}' "$0" | \ + julia --startup-file=no -E 'readchomp("/dev/fd/0") |> Meta.parse |> eval' && echo "Done"; exit +=# + """ struct TermCapability @@ -10,531 +20,777 @@ Specification of a single terminal capability. # Fields -- `short::Symbol`: The *Cap-name* of the capability -- `long::Symbol`: The name of the terminfo capability variable +- `name::Symbol`: The name of the terminfo capability variable +- `capname::Symbol`: The *Cap-name* of the capability - `description::String`: A description of the purpose of the capability See also: `TermInfo`, `TERM_FLAGS`, `TERM_NUMBERS`, and `TERM_STRINGS`. """ struct TermCapability - short::Symbol - long::Symbol + name::Symbol + capname::Symbol description::String end -# Terminfo Capabilities as of NCurses 6.3 +#=run +begin + +using Downloads + +version_info = IOBuffer() +standard_caps = IOBuffer() +user_caps = IOBuffer() + +Downloads.download("https://raw.githubusercontent.com/mirror/ncurses/master/VERSION", version_info) +Downloads.download("https://raw.githubusercontent.com/mirror/ncurses/master/include/Caps", standard_caps) +Downloads.download("https://raw.githubusercontent.com/mirror/ncurses/master/include/Caps-ncurses", user_caps) + +const TERM_FLAGS = NTuple{3, String}[] +const TERM_NUMBERS = NTuple{3, String}[] +const TERM_STRINGS = NTuple{3, String}[] +const TERM_USER = NTuple{3, String}[] + +_, ncurses_version, ncurses_date = split(read(seekstart(version_info), String)) + +for line in eachline(seekstart(standard_caps)) + startswith(line, '#') && continue + components = split(line, '\t', keepempty=false) + if length(components) ∉ 8:9 + @warn "Malformed line: $(sprint(show, line))" + continue + end + name, shortcode, type, _, _, _, _, description, _... = components + caplist = if type == "bool" TERM_FLAGS + elseif type == "num" TERM_NUMBERS + elseif type == "str" TERM_STRINGS + else + @warn "Unrecognised capability type: $type" + continue + end + push!(caplist, (name, shortcode, description)) +end + +for line in eachline(seekstart(user_caps)) + startswith(line, '#') && continue + !startswith(line, "userdef") && continue + line = line[1+ncodeunits("userdef "):end] + components = split(line, '\t', keepempty=false) + if length(components) ∉ 4:5 + @warn "Malformed line: $(sprint(show, line))" + continue + end + code, type, _, description, _... = components + if code == "xm" + components[3] == "-" || continue + description = "mouse response" + end + dtype = get(Dict("bool" => "Bool", "num" => "Int", "str" => "String"), type, nothing) + if isnothing(dtype) + @warn "Unrecognised data type: $type" + continue + end + push!(TERM_USER, (dtype, code, description)) +end + +push!(TERM_USER, ("Bool", "Tc", "tmux extension to indicate 24-bit truecolor support")) +push!(TERM_USER, ("Bool", "Su", "kitty extension to indicate styled underline support")) + +const SENTINEL = "\n## GENERATED CODE BEYOND THIS POINT ##" +const PREAMBLE = readuntil(__FILE__, SENTINEL, keep=true) + +out = IOBuffer() +write(out, PREAMBLE, "\n\n# Terminfo Capabilities as of NCurses $ncurses_version-$ncurses_date\n", + "const NCURSES_VERSION = v\"$ncurses_version.$ncurses_date\"\n") + +for (ftype, list) in [("flag", TERM_FLAGS), ("number", TERM_NUMBERS), ("string", TERM_STRINGS)] + print(out, "\n\"\"\"\n\ + Ordered list of known terminal capability $ftype fields, as of NCurses $ncurses_version-$ncurses_date.\n\ + \"\"\"\n\ + const TERM_$(uppercase(ftype))S = [") + namepad = maximum(textwidth, getindex.(list, 1)) + 1 + codepad = maximum(textwidth, getindex.(list, 2)) + 1 + for (name, shortcode, description) in list + print(out, "\n TermCapability(:", name, ',', ' '^(namepad - textwidth(name)), + ':', shortcode, ',', ' '^(codepad - textwidth(shortcode)), + '"', escape_string(description), "\"),") + end + println(out, "\n]") +end + +function getcustomalias(allterms::Vector{NTuple{3, String}}, type, short, description) + specific_aliases = Dict{String, String}( + "smxx" => ":enter_strikeout_mode", + "rmxx" => ":exit_strikeout_mode", + "Smol" => ":enter_overline_mode", + "Rmol" => ":exit_overline_mode", + "Cs" => ":set_cursor_color", + "Cr" => ":reset_cursor_color", + "Ss" => ":set_cursor_style", + "Se" => ":reset_cursor_style", + "Smulx" => ":set_underline_style", + "Su" => ":can_style_underline", + "csl" => ":clear_status_line", + "Ms" => ":set_host_clipboard", + "Tc" => ":truecolor", + "XF" => ":xterm_focus") + if startswith(short, 'k') && !occursin("keypad", description) + return ":key_" * replace(lowercase(description), r"[^a-z]" => '_') + end + return get(specific_aliases, short, "nothing") +end + +print(out, "\n\"\"\"\nTerminfo extensions that NCurses $ncurses_version-$ncurses_date is aware of.\n\"\"\"", + "\nconst TERM_USER = Dict{Tuple{DataType, Symbol}, Union{Tuple{Nothing, String}, Tuple{Symbol, String}}}(") +shortpad = maximum(textwidth, getindex.(TERM_USER, 2)) + 1 +for (type, short, description) in TERM_USER + print(out, "\n ($(rpad(type * ',', 7)) :$short)", ' '^(shortpad - textwidth(short)), + "=> (", getcustomalias(TERM_USER, type, short, description), ", \"", + escape_string(description), "\"),") +end +println(out, "\n)") + +open(io -> write(io, seekstart(out)), __FILE__, "w") + +end +=# + +## GENERATED CODE BEYOND THIS POINT ## + +# Terminfo Capabilities as of NCurses 6.4-20230311 +const NCURSES_VERSION = v"6.4.20230311" """ -Ordered list of known terminal capability flag fields, as of NCurses 6.3. +Ordered list of known terminal capability flag fields, as of NCurses 6.4-20230311. """ const TERM_FLAGS = [ - TermCapability(:bw, :auto_left_margin, "cub1 wraps from column 0 to last column"), - TermCapability(:am, :auto_right_margin, "terminal has automatic margins"), - TermCapability(:xsb, :no_esc_ctlc, "beehive (f1=escape, f2=ctrl C)"), - TermCapability(:xhp, :ceol_standout_glitch, "standout not erased by overwriting (hp)"), - TermCapability(:xenl, :eat_newline_glitch, "newline ignored after 80 cols (concept)"), - TermCapability(:eo, :erase_overstrike, "can erase overstrikes with a blank"), - TermCapability(:gn, :generic_type, "generic line type"), - TermCapability(:hc, :hard_copy, "hardcopy terminal"), - TermCapability(:km, :has_meta_key, "Has a meta key (i.e., sets 8th-bit)"), - TermCapability(:hs, :has_status_line, "has extra status line"), - TermCapability(:in, :insert_null_glitch, "insert mode distinguishes nulls"), - TermCapability(:db, :memory_below, "display may be retained below the screen"), - TermCapability(:da, :memory_above, "display may be retained above the screen"), - TermCapability(:mir, :move_insert_mode, "safe to move while in insert mode"), - TermCapability(:msgr, :move_standout_mode, "safe to move while in standout mode"), - TermCapability(:os, :over_strike, "terminal can overstrike"), - TermCapability(:eslok, :status_line_esc_ok, "escape can be used on the status line"), - TermCapability(:xt, :dest_tabs_magic_smso, "tabs destructive, magic so char (t1061)"), - TermCapability(:hz, :tilde_glitch, "cannot print ~'s (Hazeltine)"), - TermCapability(:ul, :transparent_underline, "underline character overstrikes"), - TermCapability(:xon, :xon_xoff, "terminal uses xon/xoff handshaking"), - TermCapability(:nxon, :needs_xon_xoff, "padding will not work, xon/xoff required"), - TermCapability(:mc5i, :prtr_silent, "printer will not echo on screen"), - TermCapability(:chts, :hard_cursor, "cursor is hard to see"), - TermCapability(:nrrmc, :non_rev_rmcup, "smcup does not reverse rmcup"), - TermCapability(:npc, :no_pad_char, "pad character does not exist"), - TermCapability(:ndscr, :non_dest_scroll_region, "scrolling region is non-destructive"), - TermCapability(:ccc, :can_change, "terminal can re-define existing colors"), - TermCapability(:bce, :back_color_erase, "screen erased with background color"), - TermCapability(:hls, :hue_lightness_saturation, "terminal uses only HLS color notation (Tektronix)"), - TermCapability(:xhpa, :col_addr_glitch, "only positive motion for hpa/mhpa caps"), - TermCapability(:crxm, :cr_cancels_micro_mode, "using cr turns off micro mode"), - TermCapability(:daisy, :has_print_wheel, "printer needs operator to change character set"), - TermCapability(:xvpa, :row_addr_glitch, "only positive motion for vpa/mvpa caps"), - TermCapability(:sam, :semi_auto_right_margin, "printing in last column causes cr"), - TermCapability(:cpix, :cpi_changes_res, "changing character pitch changes resolution"), - TermCapability(:lpix, :lpi_changes_res, "changing line pitch changes resolution"), - TermCapability(:OTbs, :backspaces_with_bs, "uses ^H to move left"), - TermCapability(:OTns, :crt_no_scrolling, "crt cannot scroll"), - TermCapability(:OTnc, :no_correctly_working_cr, "no way to go to start of line"), - TermCapability(:OTMT, :gnu_has_meta_key, "has meta key"), - TermCapability(:OTNL, :linefeed_is_newline, "move down with \n"), - TermCapability(:OTpt, :has_hardware_tabs, "has 8-char tabs invoked with ^I"), - TermCapability(:OTxr, :return_does_clr_eol, "return clears the line"), + TermCapability(:auto_left_margin, :bw, "cub1 wraps from column 0 to last column"), + TermCapability(:auto_right_margin, :am, "terminal has automatic margins"), + TermCapability(:no_esc_ctlc, :xsb, "beehive (f1=escape, f2=ctrl C)"), + TermCapability(:ceol_standout_glitch, :xhp, "standout not erased by overwriting (hp)"), + TermCapability(:eat_newline_glitch, :xenl, "newline ignored after 80 cols (concept)"), + TermCapability(:erase_overstrike, :eo, "can erase overstrikes with a blank"), + TermCapability(:generic_type, :gn, "generic line type"), + TermCapability(:hard_copy, :hc, "hardcopy terminal"), + TermCapability(:has_meta_key, :km, "Has a meta key (i.e., sets 8th-bit)"), + TermCapability(:has_status_line, :hs, "has extra status line"), + TermCapability(:insert_null_glitch, :in, "insert mode distinguishes nulls"), + TermCapability(:memory_above, :da, "display may be retained above the screen"), + TermCapability(:memory_below, :db, "display may be retained below the screen"), + TermCapability(:move_insert_mode, :mir, "safe to move while in insert mode"), + TermCapability(:move_standout_mode, :msgr, "safe to move while in standout mode"), + TermCapability(:over_strike, :os, "terminal can overstrike"), + TermCapability(:status_line_esc_ok, :eslok, "escape can be used on the status line"), + TermCapability(:dest_tabs_magic_smso, :xt, "tabs destructive, magic so char (t1061)"), + TermCapability(:tilde_glitch, :hz, "cannot print ~'s (Hazeltine)"), + TermCapability(:transparent_underline, :ul, "underline character overstrikes"), + TermCapability(:xon_xoff, :xon, "terminal uses xon/xoff handshaking"), + TermCapability(:needs_xon_xoff, :nxon, "padding will not work, xon/xoff required"), + TermCapability(:prtr_silent, :mc5i, "printer will not echo on screen"), + TermCapability(:hard_cursor, :chts, "cursor is hard to see"), + TermCapability(:non_rev_rmcup, :nrrmc, "smcup does not reverse rmcup"), + TermCapability(:no_pad_char, :npc, "pad character does not exist"), + TermCapability(:non_dest_scroll_region, :ndscr, "scrolling region is non-destructive"), + TermCapability(:can_change, :ccc, "terminal can re-define existing colors"), + TermCapability(:back_color_erase, :bce, "screen erased with background color"), + TermCapability(:hue_lightness_saturation, :hls, "terminal uses only HLS color notation (Tektronix)"), + TermCapability(:col_addr_glitch, :xhpa, "only positive motion for hpa/mhpa caps"), + TermCapability(:cr_cancels_micro_mode, :crxm, "using cr turns off micro mode"), + TermCapability(:has_print_wheel, :daisy, "printer needs operator to change character set"), + TermCapability(:row_addr_glitch, :xvpa, "only positive motion for vpa/mvpa caps"), + TermCapability(:semi_auto_right_margin, :sam, "printing in last column causes cr"), + TermCapability(:cpi_changes_res, :cpix, "changing character pitch changes resolution"), + TermCapability(:lpi_changes_res, :lpix, "changing line pitch changes resolution"), + TermCapability(:backspaces_with_bs, :OTbs, "uses ^H to move left"), + TermCapability(:crt_no_scrolling, :OTns, "crt cannot scroll"), + TermCapability(:no_correctly_working_cr, :OTnc, "no way to go to start of line"), + TermCapability(:gnu_has_meta_key, :OTMT, "has meta key"), + TermCapability(:linefeed_is_newline, :OTNL, "move down with \\n"), + TermCapability(:has_hardware_tabs, :OTpt, "has 8-char tabs invoked with ^I"), + TermCapability(:return_does_clr_eol, :OTxr, "return clears the line"), ] """ -Ordered list of known terminal capability number fields, as of NCurses 6.3. +Ordered list of known terminal capability number fields, as of NCurses 6.4-20230311. """ const TERM_NUMBERS = [ - TermCapability(:cols, :columns, "number of columns in a line"), - TermCapability(:it, :init_tabs, "tabs initially every # spaces"), - TermCapability(:lines, :lines, "number of lines on screen or page"), - TermCapability(:lm, :lines_of_memory, "lines of memory if > line. 0 means varies"), - TermCapability(:xmc, :magic_cookie_glitch, "number of blank characters left by smso or rmso"), - TermCapability(:pb, :padding_baud_rate, "lowest baud rate where padding needed"), - TermCapability(:vt, :virtual_terminal, "virtual terminal number (CB/unix)"), - TermCapability(:wsl, :width_status_line, "number of columns in status line"), - TermCapability(:nlab, :num_labels, "number of labels on screen"), - TermCapability(:lh, :label_height, "rows in each label"), - TermCapability(:lw, :label_width, "columns in each label"), - TermCapability(:ma, :max_attributes, "maximum combined attributes terminal can handle"), - TermCapability(:wnum, :maximum_windows, "maximum number of definable windows"), - TermCapability(:colors, :max_colors, "maximum number of colors on screen"), - TermCapability(:pairs, :max_pairs, "maximum number of color-pairs on the screen"), - TermCapability(:ncv, :no_color_video, "video attributes that cannot be used with colors"), - TermCapability(:bufsz, :buffer_capacity, "numbers of bytes buffered before printing"), - TermCapability(:spinv, :dot_vert_spacing, "spacing of pins vertically in pins per inch"), - TermCapability(:spinh, :dot_horz_spacing, "spacing of dots horizontally in dots per inch"), - TermCapability(:maddr, :max_micro_address, "maximum value in micro_..._address"), - TermCapability(:mjump, :max_micro_jump, "maximum value in parm_..._micro"), - TermCapability(:mcs, :micro_col_size, "character step size when in micro mode"), - TermCapability(:mls, :micro_line_size, "line step size when in micro mode"), - TermCapability(:npins, :number_of_pins, "numbers of pins in print-head"), - TermCapability(:orc, :output_res_char, "horizontal resolution in units per line"), - TermCapability(:orl, :output_res_line, "vertical resolution in units per line"), - TermCapability(:orhi, :output_res_horz_inch, "horizontal resolution in units per inch"), - TermCapability(:orvi, :output_res_vert_inch, "vertical resolution in units per inch"), - TermCapability(:cps, :print_rate, "print rate in characters per second"), - TermCapability(:widcs, :wide_char_size, "character step size when in double wide mode"), - TermCapability(:btns, :buttons, "number of buttons on mouse"), - TermCapability(:bitwin, :bit_image_entwining, "number of passes for each bit-image row"), - TermCapability(:bitype, :bit_image_type, "type of bit-image device"), - TermCapability(:OTug, :magic_cookie_glitch_ul, "number of blanks left by ul"), - TermCapability(:OTdC, :carriage_return_delay, "pad needed for CR"), - TermCapability(:OTdN, :new_line_delay, "pad needed for LF"), - TermCapability(:OTdB, :backspace_delay, "padding required for ^H"), - TermCapability(:OTdT, :horizontal_tab_delay, "padding required for ^I"), - TermCapability(:OTkn, :number_of_function_keys, "count of function keys"), + TermCapability(:columns, :cols, "number of columns in a line"), + TermCapability(:init_tabs, :it, "tabs initially every # spaces"), + TermCapability(:lines, :lines, "number of lines on screen or page"), + TermCapability(:lines_of_memory, :lm, "lines of memory if > line. 0 means varies"), + TermCapability(:magic_cookie_glitch, :xmc, "number of blank characters left by smso or rmso"), + TermCapability(:padding_baud_rate, :pb, "lowest baud rate where padding needed"), + TermCapability(:virtual_terminal, :vt, "virtual terminal number (CB/unix)"), + TermCapability(:width_status_line, :wsl, "number of columns in status line"), + TermCapability(:num_labels, :nlab, "number of labels on screen"), + TermCapability(:label_height, :lh, "rows in each label"), + TermCapability(:label_width, :lw, "columns in each label"), + TermCapability(:max_attributes, :ma, "maximum combined attributes terminal can handle"), + TermCapability(:maximum_windows, :wnum, "maximum number of definable windows"), + TermCapability(:max_colors, :colors, "maximum number of colors on screen"), + TermCapability(:max_pairs, :pairs, "maximum number of color-pairs on the screen"), + TermCapability(:no_color_video, :ncv, "video attributes that cannot be used with colors"), + TermCapability(:buffer_capacity, :bufsz, "numbers of bytes buffered before printing"), + TermCapability(:dot_vert_spacing, :spinv, "spacing of pins vertically in pins per inch"), + TermCapability(:dot_horz_spacing, :spinh, "spacing of dots horizontally in dots per inch"), + TermCapability(:max_micro_address, :maddr, "maximum value in micro_..._address"), + TermCapability(:max_micro_jump, :mjump, "maximum value in parm_..._micro"), + TermCapability(:micro_col_size, :mcs, "character step size when in micro mode"), + TermCapability(:micro_line_size, :mls, "line step size when in micro mode"), + TermCapability(:number_of_pins, :npins, "numbers of pins in print-head"), + TermCapability(:output_res_char, :orc, "horizontal resolution in units per line"), + TermCapability(:output_res_line, :orl, "vertical resolution in units per line"), + TermCapability(:output_res_horz_inch, :orhi, "horizontal resolution in units per inch"), + TermCapability(:output_res_vert_inch, :orvi, "vertical resolution in units per inch"), + TermCapability(:print_rate, :cps, "print rate in characters per second"), + TermCapability(:wide_char_size, :widcs, "character step size when in double wide mode"), + TermCapability(:buttons, :btns, "number of buttons on mouse"), + TermCapability(:bit_image_entwining, :bitwin, "number of passes for each bit-image row"), + TermCapability(:bit_image_type, :bitype, "type of bit-image device"), + TermCapability(:magic_cookie_glitch_ul, :OTug, "number of blanks left by ul"), + TermCapability(:carriage_return_delay, :OTdC, "pad needed for CR"), + TermCapability(:new_line_delay, :OTdN, "pad needed for LF"), + TermCapability(:backspace_delay, :OTdB, "padding required for ^H"), + TermCapability(:horizontal_tab_delay, :OTdT, "padding required for ^I"), + TermCapability(:number_of_function_keys, :OTkn, "count of function keys"), ] """ -Ordered list of known terminal capability string fields, as of NCurses 6.3. +Ordered list of known terminal capability string fields, as of NCurses 6.4-20230311. """ const TERM_STRINGS = [ - TermCapability(:cbt, :back_tab, "back tab (P)"), - TermCapability(:bel, :bell, "audible signal (bell) (P)"), - TermCapability(:cr, :carriage_return, "carriage return (P*) (P*)"), - TermCapability(:csr, :change_scroll_region, "change region to line #1 to line #2 (P)"), - TermCapability(:tbc, :clear_all_tabs, "clear all tab stops (P)"), - TermCapability(:clear, :clear_screen, "clear screen and home cursor (P*)"), - TermCapability(:el, :clr_eol, "clear to end of line (P)"), - TermCapability(:ed, :clr_eos, "clear to end of screen (P*)"), - TermCapability(:hpa, :column_address, "horizontal position #1, absolute (P)"), - TermCapability(:cmdch, :command_character, "terminal settable cmd character in prototype !?"), - TermCapability(:cup, :cursor_address, "move to row #1 columns #2"), - TermCapability(:cud1, :cursor_down, "down one line"), - TermCapability(:home, :cursor_home, "home cursor (if no cup)"), - TermCapability(:civis, :cursor_invisible, "make cursor invisible"), - TermCapability(:cub1, :cursor_left, "move left one space"), - TermCapability(:mrcup, :cursor_mem_address, "memory relative cursor addressing, move to row #1 columns #2"), - TermCapability(:cnorm, :cursor_normal, "make cursor appear normal (undo civis/cvvis)"), - TermCapability(:cuf1, :cursor_right, "non-destructive space (move right one space)"), - TermCapability(:ll, :cursor_to_ll, "last line, first column (if no cup)"), - TermCapability(:cuu1, :cursor_up, "up one line"), - TermCapability(:cvvis, :cursor_visible, "make cursor very visible"), - TermCapability(:dch1, :delete_character, "delete character (P*)"), - TermCapability(:dl1, :delete_line, "delete line (P*)"), - TermCapability(:dsl, :dis_status_line, "disable status line"), - TermCapability(:hd, :down_half_line, "half a line down"), - TermCapability(:smacs, :enter_alt_charset_mode, "start alternate character set (P)"), - TermCapability(:blink, :enter_blink_mode, "turn on blinking"), - TermCapability(:bold, :enter_bold_mode, "turn on bold (extra bright) mode"), - TermCapability(:smcup, :enter_ca_mode, "string to start programs using cup"), - TermCapability(:smdc, :enter_delete_mode, "enter delete mode"), - TermCapability(:dim, :enter_dim_mode, "turn on half-bright mode"), - TermCapability(:smir, :enter_insert_mode, "enter insert mode"), - TermCapability(:invis, :enter_secure_mode, "turn on blank mode (characters invisible)"), - TermCapability(:prot, :enter_protected_mode, "turn on protected mode"), - TermCapability(:rev, :enter_reverse_mode, "turn on reverse video mode"), - TermCapability(:smso, :enter_standout_mode, "begin standout mode"), - TermCapability(:smul, :enter_underline_mode, "begin underline mode"), - TermCapability(:ech, :erase_chars, "erase #1 characters (P)"), - TermCapability(:rmacs, :exit_alt_charset_mode, "end alternate character set (P)"), - TermCapability(:sgr0, :exit_attribute_mode, "turn off all attributes"), - TermCapability(:rmcup, :exit_ca_mode, "strings to end programs using cup"), - TermCapability(:rmdc, :exit_delete_mode, "end delete mode"), - TermCapability(:rmir, :exit_insert_mode, "exit insert mode"), - TermCapability(:rmso, :exit_standout_mode, "exit standout mode"), - TermCapability(:rmul, :exit_underline_mode, "exit underline mode"), - TermCapability(:flash, :flash_screen, "visible bell (may not move cursor)"), - TermCapability(:ff, :form_feed, "hardcopy terminal page eject (P*)"), - TermCapability(:fsl, :from_status_line, "return from status line"), - TermCapability(:is1, :init_1string, "initialization string"), - TermCapability(:is2, :init_2string, "initialization string"), - TermCapability(:is3, :init_3string, "initialization string"), - TermCapability(:if, :init_file, "name of initialization file"), - TermCapability(:ich1, :insert_character, "insert character (P)"), - TermCapability(:il1, :insert_line, "insert line (P*)"), - TermCapability(:ip, :insert_padding, "insert padding after inserted character"), - TermCapability(:kbs, :key_backspace, "backspace key"), - TermCapability(:ktbc, :key_catab, "clear-all-tabs key"), - TermCapability(:kclr, :key_clear, "clear-screen or erase key"), - TermCapability(:kctab, :key_ctab, "clear-tab key"), - TermCapability(:kdch1, :key_dc, "delete-character key"), - TermCapability(:kdl1, :key_dl, "delete-line key"), - TermCapability(:kcud1, :key_down, "down-arrow key"), - TermCapability(:krmir, :key_eic, "sent by rmir or smir in insert mode"), - TermCapability(:kel, :key_eol, "clear-to-end-of-line key"), - TermCapability(:ked, :key_eos, "clear-to-end-of-screen key"), - TermCapability(:kf0, :key_f0, "F0 function key"), - TermCapability(:kf1, :key_f1, "F1 function key"), - TermCapability(:kf10, :key_f10, "F10 function key"), - TermCapability(:kf2, :key_f2, "F2 function key"), - TermCapability(:kf3, :key_f3, "F3 function key"), - TermCapability(:kf4, :key_f4, "F4 function key"), - TermCapability(:kf5, :key_f5, "F5 function key"), - TermCapability(:kf6, :key_f6, "F6 function key"), - TermCapability(:kf7, :key_f7, "F7 function key"), - TermCapability(:kf8, :key_f8, "F8 function key"), - TermCapability(:kf9, :key_f9, "F9 function key"), - TermCapability(:khome, :key_home, "home key"), - TermCapability(:kich1, :key_ic, "insert-character key"), - TermCapability(:kil1, :key_il, "insert-line key"), - TermCapability(:kcub1, :key_left, "left-arrow key"), - TermCapability(:kll, :key_ll, "lower-left key (home down)"), - TermCapability(:knp, :key_npage, "next-page key"), - TermCapability(:kpp, :key_ppage, "previous-page key"), - TermCapability(:kcuf1, :key_right, "right-arrow key"), - TermCapability(:kind, :key_sf, "scroll-forward key"), - TermCapability(:kri, :key_sr, "scroll-backward key"), - TermCapability(:khts, :key_stab, "set-tab key"), - TermCapability(:kcuu1, :key_up, "up-arrow key"), - TermCapability(:rmkx, :keypad_local, "leave 'keyboard_transmit' mode"), - TermCapability(:smkx, :keypad_xmit, "enter 'keyboard_transmit' mode"), - TermCapability(:lf0, :lab_f0, "label on function key f0 if not f0"), - TermCapability(:lf1, :lab_f1, "label on function key f1 if not f1"), - TermCapability(:lf10, :lab_f10, "label on function key f10 if not f10"), - TermCapability(:lf2, :lab_f2, "label on function key f2 if not f2"), - TermCapability(:lf3, :lab_f3, "label on function key f3 if not f3"), - TermCapability(:lf4, :lab_f4, "label on function key f4 if not f4"), - TermCapability(:lf5, :lab_f5, "label on function key f5 if not f5"), - TermCapability(:lf6, :lab_f6, "label on function key f6 if not f6"), - TermCapability(:lf7, :lab_f7, "label on function key f7 if not f7"), - TermCapability(:lf8, :lab_f8, "label on function key f8 if not f8"), - TermCapability(:lf9, :lab_f9, "label on function key f9 if not f9"), - TermCapability(:rmm, :meta_off, "turn off meta mode"), - TermCapability(:smm, :meta_on, "turn on meta mode (8th-bit on)"), - TermCapability(:nel, :newline, "newline (behave like cr followed by lf)"), - TermCapability(:pad, :pad_char, "padding char (instead of null)"), - TermCapability(:dch, :parm_dch, "delete #1 characters (P*)"), - TermCapability(:dl, :parm_delete_line, "delete #1 lines (P*)"), - TermCapability(:cud, :parm_down_cursor, "down #1 lines (P*)"), - TermCapability(:ich, :parm_ich, "insert #1 characters (P*)"), - TermCapability(:indn, :parm_index, "scroll forward #1 lines (P)"), - TermCapability(:il, :parm_insert_line, "insert #1 lines (P*)"), - TermCapability(:cub, :parm_left_cursor, "move #1 characters to the left (P)"), - TermCapability(:cuf, :parm_right_cursor, "move #1 characters to the right (P*)"), - TermCapability(:rin, :parm_rindex, "scroll back #1 lines (P)"), - TermCapability(:cuu, :parm_up_cursor, "up #1 lines (P*)"), - TermCapability(:pfkey, :pkey_key, "program function key #1 to type string #2"), - TermCapability(:pfloc, :pkey_local, "program function key #1 to execute string #2"), - TermCapability(:pfx, :pkey_xmit, "program function key #1 to transmit string #2"), - TermCapability(:mc0, :print_screen, "print contents of screen"), - TermCapability(:mc4, :prtr_off, "turn off printer"), - TermCapability(:mc5, :prtr_on, "turn on printer"), - TermCapability(:rep, :repeat_char, "repeat char #1 #2 times (P*)"), - TermCapability(:rs1, :reset_1string, "reset string"), - TermCapability(:rs2, :reset_2string, "reset string"), - TermCapability(:rs3, :reset_3string, "reset string"), - TermCapability(:rf, :reset_file, "name of reset file"), - TermCapability(:rc, :restore_cursor, "restore cursor to position of last save_cursor"), - TermCapability(:vpa, :row_address, "vertical position #1 absolute (P)"), - TermCapability(:sc, :save_cursor, "save current cursor position (P)"), - TermCapability(:ind, :scroll_forward, "scroll text up (P)"), - TermCapability(:ri, :scroll_reverse, "scroll text down (P)"), - TermCapability(:sgr, :set_attributes, "define video attributes #1-#9 (PG9)"), - TermCapability(:hts, :set_tab, "set a tab in every row, current columns"), - TermCapability(:wind, :set_window, "current window is lines #1-#2 cols #3-#4"), - TermCapability(:ht, :tab, "tab to next 8-space hardware tab stop"), - TermCapability(:tsl, :to_status_line, "move to status line, column #1"), - TermCapability(:uc, :underline_char, "underline char and move past it"), - TermCapability(:hu, :up_half_line, "half a line up"), - TermCapability(:iprog, :init_prog, "path name of program for initialization"), - TermCapability(:ka1, :key_a1, "upper left of keypad"), - TermCapability(:ka3, :key_a3, "upper right of keypad"), - TermCapability(:kb2, :key_b2, "center of keypad"), - TermCapability(:kc1, :key_c1, "lower left of keypad"), - TermCapability(:kc3, :key_c3, "lower right of keypad"), - TermCapability(:mc5p, :prtr_non, "turn on printer for #1 bytes"), - TermCapability(:rmp, :char_padding, "like ip but when in insert mode"), - TermCapability(:acsc, :acs_chars, "graphics charset pairs, based on vt100"), - TermCapability(:pln, :plab_norm, "program label #1 to show string #2"), - TermCapability(:kcbt, :key_btab, "back-tab key"), - TermCapability(:smxon, :enter_xon_mode, "turn on xon/xoff handshaking"), - TermCapability(:rmxon, :exit_xon_mode, "turn off xon/xoff handshaking"), - TermCapability(:smam, :enter_am_mode, "turn on automatic margins"), - TermCapability(:rmam, :exit_am_mode, "turn off automatic margins"), - TermCapability(:xonc, :xon_character, "XON character"), - TermCapability(:xoffc, :xoff_character, "XOFF character"), - TermCapability(:enacs, :ena_acs, "enable alternate char set"), - TermCapability(:smln, :label_on, "turn on soft labels"), - TermCapability(:rmln, :label_off, "turn off soft labels"), - TermCapability(:kbeg, :key_beg, "begin key"), - TermCapability(:kcan, :key_cancel, "cancel key"), - TermCapability(:kclo, :key_close, "close key"), - TermCapability(:kcmd, :key_command, "command key"), - TermCapability(:kcpy, :key_copy, "copy key"), - TermCapability(:kcrt, :key_create, "create key"), - TermCapability(:kend, :key_end, "end key"), - TermCapability(:kent, :key_enter, "enter/send key"), - TermCapability(:kext, :key_exit, "exit key"), - TermCapability(:kfnd, :key_find, "find key"), - TermCapability(:khlp, :key_help, "help key"), - TermCapability(:kmrk, :key_mark, "mark key"), - TermCapability(:kmsg, :key_message, "message key"), - TermCapability(:kmov, :key_move, "move key"), - TermCapability(:knxt, :key_next, "next key"), - TermCapability(:kopn, :key_open, "open key"), - TermCapability(:kopt, :key_options, "options key"), - TermCapability(:kprv, :key_previous, "previous key"), - TermCapability(:kprt, :key_print, "print key"), - TermCapability(:krdo, :key_redo, "redo key"), - TermCapability(:kref, :key_reference, "reference key"), - TermCapability(:krfr, :key_refresh, "refresh key"), - TermCapability(:krpl, :key_replace, "replace key"), - TermCapability(:krst, :key_restart, "restart key"), - TermCapability(:kres, :key_resume, "resume key"), - TermCapability(:ksav, :key_save, "save key"), - TermCapability(:kspd, :key_suspend, "suspend key"), - TermCapability(:kund, :key_undo, "undo key"), - TermCapability(:kBEG, :key_sbeg, "shifted begin key"), - TermCapability(:kCAN, :key_scancel, "shifted cancel key"), - TermCapability(:kCMD, :key_scommand, "shifted command key"), - TermCapability(:kCPY, :key_scopy, "shifted copy key"), - TermCapability(:kCRT, :key_screate, "shifted create key"), - TermCapability(:kDC, :key_sdc, "shifted delete-character key"), - TermCapability(:kDL, :key_sdl, "shifted delete-line key"), - TermCapability(:kslt, :key_select, "select key"), - TermCapability(:kEND, :key_send, "shifted end key"), - TermCapability(:kEOL, :key_seol, "shifted clear-to-end-of-line key"), - TermCapability(:kEXT, :key_sexit, "shifted exit key"), - TermCapability(:kFND, :key_sfind, "shifted find key"), - TermCapability(:kHLP, :key_shelp, "shifted help key"), - TermCapability(:kHOM, :key_shome, "shifted home key"), - TermCapability(:kIC, :key_sic, "shifted insert-character key"), - TermCapability(:kLFT, :key_sleft, "shifted left-arrow key"), - TermCapability(:kMSG, :key_smessage, "shifted message key"), - TermCapability(:kMOV, :key_smove, "shifted move key"), - TermCapability(:kNXT, :key_snext, "shifted next key"), - TermCapability(:kOPT, :key_soptions, "shifted options key"), - TermCapability(:kPRV, :key_sprevious, "shifted previous key"), - TermCapability(:kPRT, :key_sprint, "shifted print key"), - TermCapability(:kRDO, :key_sredo, "shifted redo key"), - TermCapability(:kRPL, :key_sreplace, "shifted replace key"), - TermCapability(:kRIT, :key_sright, "shifted right-arrow key"), - TermCapability(:kRES, :key_srsume, "shifted resume key"), - TermCapability(:kSAV, :key_ssave, "shifted save key"), - TermCapability(:kSPD, :key_ssuspend, "shifted suspend key"), - TermCapability(:kUND, :key_sundo, "shifted undo key"), - TermCapability(:rfi, :req_for_input, "send next input char (for ptys)"), - TermCapability(:kf11, :key_f11, "F11 function key"), - TermCapability(:kf12, :key_f12, "F12 function key"), - TermCapability(:kf13, :key_f13, "F13 function key"), - TermCapability(:kf14, :key_f14, "F14 function key"), - TermCapability(:kf15, :key_f15, "F15 function key"), - TermCapability(:kf16, :key_f16, "F16 function key"), - TermCapability(:kf17, :key_f17, "F17 function key"), - TermCapability(:kf18, :key_f18, "F18 function key"), - TermCapability(:kf19, :key_f19, "F19 function key"), - TermCapability(:kf20, :key_f20, "F20 function key"), - TermCapability(:kf21, :key_f21, "F21 function key"), - TermCapability(:kf22, :key_f22, "F22 function key"), - TermCapability(:kf23, :key_f23, "F23 function key"), - TermCapability(:kf24, :key_f24, "F24 function key"), - TermCapability(:kf25, :key_f25, "F25 function key"), - TermCapability(:kf26, :key_f26, "F26 function key"), - TermCapability(:kf27, :key_f27, "F27 function key"), - TermCapability(:kf28, :key_f28, "F28 function key"), - TermCapability(:kf29, :key_f29, "F29 function key"), - TermCapability(:kf30, :key_f30, "F30 function key"), - TermCapability(:kf31, :key_f31, "F31 function key"), - TermCapability(:kf32, :key_f32, "F32 function key"), - TermCapability(:kf33, :key_f33, "F33 function key"), - TermCapability(:kf34, :key_f34, "F34 function key"), - TermCapability(:kf35, :key_f35, "F35 function key"), - TermCapability(:kf36, :key_f36, "F36 function key"), - TermCapability(:kf37, :key_f37, "F37 function key"), - TermCapability(:kf38, :key_f38, "F38 function key"), - TermCapability(:kf39, :key_f39, "F39 function key"), - TermCapability(:kf40, :key_f40, "F40 function key"), - TermCapability(:kf41, :key_f41, "F41 function key"), - TermCapability(:kf42, :key_f42, "F42 function key"), - TermCapability(:kf43, :key_f43, "F43 function key"), - TermCapability(:kf44, :key_f44, "F44 function key"), - TermCapability(:kf45, :key_f45, "F45 function key"), - TermCapability(:kf46, :key_f46, "F46 function key"), - TermCapability(:kf47, :key_f47, "F47 function key"), - TermCapability(:kf48, :key_f48, "F48 function key"), - TermCapability(:kf49, :key_f49, "F49 function key"), - TermCapability(:kf50, :key_f50, "F50 function key"), - TermCapability(:kf51, :key_f51, "F51 function key"), - TermCapability(:kf52, :key_f52, "F52 function key"), - TermCapability(:kf53, :key_f53, "F53 function key"), - TermCapability(:kf54, :key_f54, "F54 function key"), - TermCapability(:kf55, :key_f55, "F55 function key"), - TermCapability(:kf56, :key_f56, "F56 function key"), - TermCapability(:kf57, :key_f57, "F57 function key"), - TermCapability(:kf58, :key_f58, "F58 function key"), - TermCapability(:kf59, :key_f59, "F59 function key"), - TermCapability(:kf60, :key_f60, "F60 function key"), - TermCapability(:kf61, :key_f61, "F61 function key"), - TermCapability(:kf62, :key_f62, "F62 function key"), - TermCapability(:kf63, :key_f63, "F63 function key"), - TermCapability(:el1, :clr_bol, "Clear to beginning of line"), - TermCapability(:mgc, :clear_margins, "clear right and left soft margins"), - TermCapability(:smgl, :set_left_margin, "set left soft margin at current column. (ML is not in BSD termcap)."), - TermCapability(:smgr, :set_right_margin, "set right soft margin at current column"), - TermCapability(:fln, :label_format, "label format"), - TermCapability(:sclk, :set_clock, "set clock, #1 hrs #2 mins #3 secs"), - TermCapability(:dclk, :display_clock, "display clock"), - TermCapability(:rmclk, :remove_clock, "remove clock"), - TermCapability(:cwin, :create_window, "define a window #1 from #2, #3 to #4, #5"), - TermCapability(:wingo, :goto_window, "go to window #1"), - TermCapability(:hup, :hangup, "hang-up phone"), - TermCapability(:dial, :dial_phone, "dial number #1"), - TermCapability(:qdial, :quick_dial, "dial number #1 without checking"), - TermCapability(:tone, :tone, "select touch tone dialing"), - TermCapability(:pulse, :pulse, "select pulse dialing"), - TermCapability(:hook, :flash_hook, "flash switch hook"), - TermCapability(:pause, :fixed_pause, "pause for 2-3 seconds"), - TermCapability(:wait, :wait_tone, "wait for dial-tone"), - TermCapability(:u0, :user0, "User string #0"), - TermCapability(:u1, :user1, "User string #1"), - TermCapability(:u2, :user2, "User string #2"), - TermCapability(:u3, :user3, "User string #3"), - TermCapability(:u4, :user4, "User string #4"), - TermCapability(:u5, :user5, "User string #5"), - TermCapability(:u6, :user6, "User string #6"), - TermCapability(:u7, :user7, "User string #7"), - TermCapability(:u8, :user8, "User string #8"), - TermCapability(:u9, :user9, "User string #9"), - TermCapability(:op, :orig_pair, "Set default pair to its original value"), - TermCapability(:oc, :orig_colors, "Set all color pairs to the original ones"), - TermCapability(:initc, :initialize_color, "Initialize color #1 to (#2, #3, #4)"), - TermCapability(:initp, :initialize_pair, "Initialize color pair #1 to fg=(#2, #3, #4), bg=(#5,#6,#7)"), - TermCapability(:scp, :set_color_pair, "Set current color pair to #1"), - TermCapability(:setf, :set_foreground, "Set foreground color #1"), - TermCapability(:setb, :set_background, "Set background color #1"), - TermCapability(:cpi, :change_char_pitch, "Change number of characters per inch to #1"), - TermCapability(:lpi, :change_line_pitch, "Change number of lines per inch to #1"), - TermCapability(:chr, :change_res_horz, "Change horizontal resolution to #1"), - TermCapability(:cvr, :change_res_vert, "Change vertical resolution to #1"), - TermCapability(:defc, :define_char, "Define a character #1, #2 dots wide, descender #3"), - TermCapability(:swidm, :enter_doublewide_mode, "Enter double-wide mode"), - TermCapability(:sdrfq, :enter_draft_quality, "Enter draft-quality mode"), - TermCapability(:sitm, :enter_italics_mode, "Enter italic mode"), - TermCapability(:slm, :enter_leftward_mode, "Start leftward carriage motion"), - TermCapability(:smicm, :enter_micro_mode, "Start micro-motion mode"), - TermCapability(:snlq, :enter_near_letter_quality, "Enter NLQ mode"), - TermCapability(:snrmq, :enter_normal_quality, "Enter normal-quality mode"), - TermCapability(:sshm, :enter_shadow_mode, "Enter shadow-print mode"), - TermCapability(:ssubm, :enter_subscript_mode, "Enter subscript mode"), - TermCapability(:ssupm, :enter_superscript_mode, "Enter superscript mode"), - TermCapability(:sum, :enter_upward_mode, "Start upward carriage motion"), - TermCapability(:rwidm, :exit_doublewide_mode, "End double-wide mode"), - TermCapability(:ritm, :exit_italics_mode, "End italic mode"), - TermCapability(:rlm, :exit_leftward_mode, "End left-motion mode"), - TermCapability(:rmicm, :exit_micro_mode, "End micro-motion mode"), - TermCapability(:rshm, :exit_shadow_mode, "End shadow-print mode"), - TermCapability(:rsubm, :exit_subscript_mode, "End subscript mode"), - TermCapability(:rsupm, :exit_superscript_mode, "End superscript mode"), - TermCapability(:rum, :exit_upward_mode, "End reverse character motion"), - TermCapability(:mhpa, :micro_column_address, "Like column_address in micro mode"), - TermCapability(:mcud1, :micro_down, "Like cursor_down in micro mode"), - TermCapability(:mcub1, :micro_left, "Like cursor_left in micro mode"), - TermCapability(:mcuf1, :micro_right, "Like cursor_right in micro mode"), - TermCapability(:mvpa, :micro_row_address, "Like row_address #1 in micro mode"), - TermCapability(:mcuu1, :micro_up, "Like cursor_up in micro mode"), - TermCapability(:porder, :order_of_pins, "Match software bits to print-head pins"), - TermCapability(:mcud, :parm_down_micro, "Like parm_down_cursor in micro mode"), - TermCapability(:mcub, :parm_left_micro, "Like parm_left_cursor in micro mode"), - TermCapability(:mcuf, :parm_right_micro, "Like parm_right_cursor in micro mode"), - TermCapability(:mcuu, :parm_up_micro, "Like parm_up_cursor in micro mode"), - TermCapability(:scs, :select_char_set, "Select character set, #1"), - TermCapability(:smgb, :set_bottom_margin, "Set bottom margin at current line"), - TermCapability(:smgbp, :set_bottom_margin_parm, "Set bottom margin at line #1 or (if smgtp is not given) #2 lines from bottom"), - TermCapability(:smglp, :set_left_margin_parm, "Set left (right) margin at column #1"), - TermCapability(:smgrp, :set_right_margin_parm, "Set right margin at column #1"), - TermCapability(:smgt, :set_top_margin, "Set top margin at current line"), - TermCapability(:smgtp, :set_top_margin_parm, "Set top (bottom) margin at row #1"), - TermCapability(:sbim, :start_bit_image, "Start printing bit image graphics"), - TermCapability(:scsd, :start_char_set_def, "Start character set definition #1, with #2 characters in the set"), - TermCapability(:rbim, :stop_bit_image, "Stop printing bit image graphics"), - TermCapability(:rcsd, :stop_char_set_def, "End definition of character set #1"), - TermCapability(:subcs, :subscript_characters, "List of subscriptable characters"), - TermCapability(:supcs, :superscript_characters, "List of superscriptable characters"), - TermCapability(:docr, :these_cause_cr, "Printing any of these characters causes CR"), - TermCapability(:zerom, :zero_motion, "No motion for subsequent character"), - TermCapability(:csnm, :char_set_names, "Produce #1'th item from list of character set names"), - TermCapability(:kmous, :key_mouse, "Mouse event has occurred"), - TermCapability(:minfo, :mouse_info, "Mouse status information"), - TermCapability(:reqmp, :req_mouse_pos, "Request mouse position"), - TermCapability(:getm, :get_mouse, "Curses should get button events, parameter #1 not documented."), - TermCapability(:setaf, :set_a_foreground, "Set foreground color to #1, using ANSI escape"), - TermCapability(:setab, :set_a_background, "Set background color to #1, using ANSI escape"), - TermCapability(:pfxl, :pkey_plab, "Program function key #1 to type string #2 and show string #3"), - TermCapability(:devt, :device_type, "Indicate language/codeset support"), - TermCapability(:csin, :code_set_init, "Init sequence for multiple codesets"), - TermCapability(:s0ds, :set0_des_seq, "Shift to codeset 0 (EUC set 0, ASCII)"), - TermCapability(:s1ds, :set1_des_seq, "Shift to codeset 1"), - TermCapability(:s2ds, :set2_des_seq, "Shift to codeset 2"), - TermCapability(:s3ds, :set3_des_seq, "Shift to codeset 3"), - TermCapability(:smglr, :set_lr_margin, "Set both left and right margins to #1, #2. (ML is not in BSD termcap)."), - TermCapability(:smgtb, :set_tb_margin, "Sets both top and bottom margins to #1, #2"), - TermCapability(:birep, :bit_image_repeat, "Repeat bit image cell #1 #2 times"), - TermCapability(:binel, :bit_image_newline, "Move to next row of the bit image"), - TermCapability(:bicr, :bit_image_carriage_return, "Move to beginning of same row"), - TermCapability(:colornm, :color_names, "Give name for color #1"), - TermCapability(:defbi, :define_bit_image_region, "Define rectangular bit image region"), - TermCapability(:endbi, :end_bit_image_region, "End a bit-image region"), - TermCapability(:setcolor, :set_color_band, "Change to ribbon color #1"), - TermCapability(:slines, :set_page_length, "Set page length to #1 lines"), - TermCapability(:dispc, :display_pc_char, "Display PC character #1"), - TermCapability(:smpch, :enter_pc_charset_mode, "Enter PC character display mode"), - TermCapability(:rmpch, :exit_pc_charset_mode, "Exit PC character display mode"), - TermCapability(:smsc, :enter_scancode_mode, "Enter PC scancode mode"), - TermCapability(:rmsc, :exit_scancode_mode, "Exit PC scancode mode"), - TermCapability(:pctrm, :pc_term_options, "PC terminal options"), - TermCapability(:scesc, :scancode_escape, "Escape for scancode emulation"), - TermCapability(:scesa, :alt_scancode_esc, "Alternate escape for scancode emulation"), - TermCapability(:ehhlm, :enter_horizontal_hl_mode, "Enter horizontal highlight mode"), - TermCapability(:elhlm, :enter_left_hl_mode, "Enter left highlight mode"), - TermCapability(:elohlm, :enter_low_hl_mode, "Enter low highlight mode"), - TermCapability(:erhlm, :enter_right_hl_mode, "Enter right highlight mode"), - TermCapability(:ethlm, :enter_top_hl_mode, "Enter top highlight mode"), - TermCapability(:evhlm, :enter_vertical_hl_mode, "Enter vertical highlight mode"), - TermCapability(:sgr1, :set_a_attributes, "Define second set of video attributes #1-#6"), - TermCapability(:slength, :set_pglen_inch, "Set page length to #1 hundredth of an inch (some implementations use sL for termcap)."), - TermCapability(:OTi2, :termcap_init2, "secondary initialization string"), - TermCapability(:OTrs, :termcap_reset, "terminal reset string"), - TermCapability(:OTnl, :linefeed_if_not_lf, "use to move down"), - TermCapability(:OTbs, :backspaces_with_bs, "uses ^H to move left"), - TermCapability(:OTko, :other_non_function_keys, "list of self-mapped keycaps"), - TermCapability(:OTma, :arrow_key_map, "map motion-keys for vi version 2"), - TermCapability(:OTG2, :acs_ulcorner, "single upper left"), - TermCapability(:OTG3, :acs_llcorner, "single lower left"), - TermCapability(:OTG1, :acs_urcorner, "single upper right"), - TermCapability(:OTG4, :acs_lrcorner, "single lower right"), - TermCapability(:OTGR, :acs_ltee, "tee pointing right"), - TermCapability(:OTGL, :acs_rtee, "tee pointing left"), - TermCapability(:OTGU, :acs_btee, "tee pointing up"), - TermCapability(:OTGD, :acs_ttee, "tee pointing down"), - TermCapability(:OTGH, :acs_hline, "single horizontal line"), - TermCapability(:OTGV, :acs_vline, "single vertical line"), - TermCapability(:OTGC, :acs_plus, "single intersection"), - TermCapability(:meml, :memory_lock, "lock memory above cursor"), - TermCapability(:memu, :memory_unlock, "unlock memory"), - TermCapability(:box1, :box_chars_1, "box characters primary set"), + TermCapability(:back_tab, :cbt, "back tab (P)"), + TermCapability(:bell, :bel, "audible signal (bell) (P)"), + TermCapability(:carriage_return, :cr, "carriage return (P*) (P*)"), + TermCapability(:change_scroll_region, :csr, "change region to line #1 to line #2 (P)"), + TermCapability(:clear_all_tabs, :tbc, "clear all tab stops (P)"), + TermCapability(:clear_screen, :clear, "clear screen and home cursor (P*)"), + TermCapability(:clr_eol, :el, "clear to end of line (P)"), + TermCapability(:clr_eos, :ed, "clear to end of screen (P*)"), + TermCapability(:column_address, :hpa, "horizontal position #1, absolute (P)"), + TermCapability(:command_character, :cmdch, "terminal settable cmd character in prototype !?"), + TermCapability(:cursor_address, :cup, "move to row #1 columns #2"), + TermCapability(:cursor_down, :cud1, "down one line"), + TermCapability(:cursor_home, :home, "home cursor (if no cup)"), + TermCapability(:cursor_invisible, :civis, "make cursor invisible"), + TermCapability(:cursor_left, :cub1, "move left one space"), + TermCapability(:cursor_mem_address, :mrcup, "memory relative cursor addressing, move to row #1 columns #2"), + TermCapability(:cursor_normal, :cnorm, "make cursor appear normal (undo civis/cvvis)"), + TermCapability(:cursor_right, :cuf1, "non-destructive space (move right one space)"), + TermCapability(:cursor_to_ll, :ll, "last line, first column (if no cup)"), + TermCapability(:cursor_up, :cuu1, "up one line"), + TermCapability(:cursor_visible, :cvvis, "make cursor very visible"), + TermCapability(:delete_character, :dch1, "delete character (P*)"), + TermCapability(:delete_line, :dl1, "delete line (P*)"), + TermCapability(:dis_status_line, :dsl, "disable status line"), + TermCapability(:down_half_line, :hd, "half a line down"), + TermCapability(:enter_alt_charset_mode, :smacs, "start alternate character set (P)"), + TermCapability(:enter_blink_mode, :blink, "turn on blinking"), + TermCapability(:enter_bold_mode, :bold, "turn on bold (extra bright) mode"), + TermCapability(:enter_ca_mode, :smcup, "string to start programs using cup"), + TermCapability(:enter_delete_mode, :smdc, "enter delete mode"), + TermCapability(:enter_dim_mode, :dim, "turn on half-bright mode"), + TermCapability(:enter_insert_mode, :smir, "enter insert mode"), + TermCapability(:enter_secure_mode, :invis, "turn on blank mode (characters invisible)"), + TermCapability(:enter_protected_mode, :prot, "turn on protected mode"), + TermCapability(:enter_reverse_mode, :rev, "turn on reverse video mode"), + TermCapability(:enter_standout_mode, :smso, "begin standout mode"), + TermCapability(:enter_underline_mode, :smul, "begin underline mode"), + TermCapability(:erase_chars, :ech, "erase #1 characters (P)"), + TermCapability(:exit_alt_charset_mode, :rmacs, "end alternate character set (P)"), + TermCapability(:exit_attribute_mode, :sgr0, "turn off all attributes"), + TermCapability(:exit_ca_mode, :rmcup, "strings to end programs using cup"), + TermCapability(:exit_delete_mode, :rmdc, "end delete mode"), + TermCapability(:exit_insert_mode, :rmir, "exit insert mode"), + TermCapability(:exit_standout_mode, :rmso, "exit standout mode"), + TermCapability(:exit_underline_mode, :rmul, "exit underline mode"), + TermCapability(:flash_screen, :flash, "visible bell (may not move cursor)"), + TermCapability(:form_feed, :ff, "hardcopy terminal page eject (P*)"), + TermCapability(:from_status_line, :fsl, "return from status line"), + TermCapability(:init_1string, :is1, "initialization string"), + TermCapability(:init_2string, :is2, "initialization string"), + TermCapability(:init_3string, :is3, "initialization string"), + TermCapability(:init_file, :if, "name of initialization file"), + TermCapability(:insert_character, :ich1, "insert character (P)"), + TermCapability(:insert_line, :il1, "insert line (P*)"), + TermCapability(:insert_padding, :ip, "insert padding after inserted character"), + TermCapability(:key_backspace, :kbs, "backspace key"), + TermCapability(:key_catab, :ktbc, "clear-all-tabs key"), + TermCapability(:key_clear, :kclr, "clear-screen or erase key"), + TermCapability(:key_ctab, :kctab, "clear-tab key"), + TermCapability(:key_dc, :kdch1, "delete-character key"), + TermCapability(:key_dl, :kdl1, "delete-line key"), + TermCapability(:key_down, :kcud1, "down-arrow key"), + TermCapability(:key_eic, :krmir, "sent by rmir or smir in insert mode"), + TermCapability(:key_eol, :kel, "clear-to-end-of-line key"), + TermCapability(:key_eos, :ked, "clear-to-end-of-screen key"), + TermCapability(:key_f0, :kf0, "F0 function key"), + TermCapability(:key_f1, :kf1, "F1 function key"), + TermCapability(:key_f10, :kf10, "F10 function key"), + TermCapability(:key_f2, :kf2, "F2 function key"), + TermCapability(:key_f3, :kf3, "F3 function key"), + TermCapability(:key_f4, :kf4, "F4 function key"), + TermCapability(:key_f5, :kf5, "F5 function key"), + TermCapability(:key_f6, :kf6, "F6 function key"), + TermCapability(:key_f7, :kf7, "F7 function key"), + TermCapability(:key_f8, :kf8, "F8 function key"), + TermCapability(:key_f9, :kf9, "F9 function key"), + TermCapability(:key_home, :khome, "home key"), + TermCapability(:key_ic, :kich1, "insert-character key"), + TermCapability(:key_il, :kil1, "insert-line key"), + TermCapability(:key_left, :kcub1, "left-arrow key"), + TermCapability(:key_ll, :kll, "lower-left key (home down)"), + TermCapability(:key_npage, :knp, "next-page key"), + TermCapability(:key_ppage, :kpp, "previous-page key"), + TermCapability(:key_right, :kcuf1, "right-arrow key"), + TermCapability(:key_sf, :kind, "scroll-forward key"), + TermCapability(:key_sr, :kri, "scroll-backward key"), + TermCapability(:key_stab, :khts, "set-tab key"), + TermCapability(:key_up, :kcuu1, "up-arrow key"), + TermCapability(:keypad_local, :rmkx, "leave 'keyboard_transmit' mode"), + TermCapability(:keypad_xmit, :smkx, "enter 'keyboard_transmit' mode"), + TermCapability(:lab_f0, :lf0, "label on function key f0 if not f0"), + TermCapability(:lab_f1, :lf1, "label on function key f1 if not f1"), + TermCapability(:lab_f10, :lf10, "label on function key f10 if not f10"), + TermCapability(:lab_f2, :lf2, "label on function key f2 if not f2"), + TermCapability(:lab_f3, :lf3, "label on function key f3 if not f3"), + TermCapability(:lab_f4, :lf4, "label on function key f4 if not f4"), + TermCapability(:lab_f5, :lf5, "label on function key f5 if not f5"), + TermCapability(:lab_f6, :lf6, "label on function key f6 if not f6"), + TermCapability(:lab_f7, :lf7, "label on function key f7 if not f7"), + TermCapability(:lab_f8, :lf8, "label on function key f8 if not f8"), + TermCapability(:lab_f9, :lf9, "label on function key f9 if not f9"), + TermCapability(:meta_off, :rmm, "turn off meta mode"), + TermCapability(:meta_on, :smm, "turn on meta mode (8th-bit on)"), + TermCapability(:newline, :nel, "newline (behave like cr followed by lf)"), + TermCapability(:pad_char, :pad, "padding char (instead of null)"), + TermCapability(:parm_dch, :dch, "delete #1 characters (P*)"), + TermCapability(:parm_delete_line, :dl, "delete #1 lines (P*)"), + TermCapability(:parm_down_cursor, :cud, "down #1 lines (P*)"), + TermCapability(:parm_ich, :ich, "insert #1 characters (P*)"), + TermCapability(:parm_index, :indn, "scroll forward #1 lines (P)"), + TermCapability(:parm_insert_line, :il, "insert #1 lines (P*)"), + TermCapability(:parm_left_cursor, :cub, "move #1 characters to the left (P)"), + TermCapability(:parm_right_cursor, :cuf, "move #1 characters to the right (P*)"), + TermCapability(:parm_rindex, :rin, "scroll back #1 lines (P)"), + TermCapability(:parm_up_cursor, :cuu, "up #1 lines (P*)"), + TermCapability(:pkey_key, :pfkey, "program function key #1 to type string #2"), + TermCapability(:pkey_local, :pfloc, "program function key #1 to execute string #2"), + TermCapability(:pkey_xmit, :pfx, "program function key #1 to transmit string #2"), + TermCapability(:print_screen, :mc0, "print contents of screen"), + TermCapability(:prtr_off, :mc4, "turn off printer"), + TermCapability(:prtr_on, :mc5, "turn on printer"), + TermCapability(:repeat_char, :rep, "repeat char #1 #2 times (P*)"), + TermCapability(:reset_1string, :rs1, "reset string"), + TermCapability(:reset_2string, :rs2, "reset string"), + TermCapability(:reset_3string, :rs3, "reset string"), + TermCapability(:reset_file, :rf, "name of reset file"), + TermCapability(:restore_cursor, :rc, "restore cursor to position of last save_cursor"), + TermCapability(:row_address, :vpa, "vertical position #1 absolute (P)"), + TermCapability(:save_cursor, :sc, "save current cursor position (P)"), + TermCapability(:scroll_forward, :ind, "scroll text up (P)"), + TermCapability(:scroll_reverse, :ri, "scroll text down (P)"), + TermCapability(:set_attributes, :sgr, "define video attributes #1-#9 (PG9)"), + TermCapability(:set_tab, :hts, "set a tab in every row, current columns"), + TermCapability(:set_window, :wind, "current window is lines #1-#2 cols #3-#4"), + TermCapability(:tab, :ht, "tab to next 8-space hardware tab stop"), + TermCapability(:to_status_line, :tsl, "move to status line, column #1"), + TermCapability(:underline_char, :uc, "underline char and move past it"), + TermCapability(:up_half_line, :hu, "half a line up"), + TermCapability(:init_prog, :iprog, "path name of program for initialization"), + TermCapability(:key_a1, :ka1, "upper left of keypad"), + TermCapability(:key_a3, :ka3, "upper right of keypad"), + TermCapability(:key_b2, :kb2, "center of keypad"), + TermCapability(:key_c1, :kc1, "lower left of keypad"), + TermCapability(:key_c3, :kc3, "lower right of keypad"), + TermCapability(:prtr_non, :mc5p, "turn on printer for #1 bytes"), + TermCapability(:char_padding, :rmp, "like ip but when in insert mode"), + TermCapability(:acs_chars, :acsc, "graphics charset pairs, based on vt100"), + TermCapability(:plab_norm, :pln, "program label #1 to show string #2"), + TermCapability(:key_btab, :kcbt, "back-tab key"), + TermCapability(:enter_xon_mode, :smxon, "turn on xon/xoff handshaking"), + TermCapability(:exit_xon_mode, :rmxon, "turn off xon/xoff handshaking"), + TermCapability(:enter_am_mode, :smam, "turn on automatic margins"), + TermCapability(:exit_am_mode, :rmam, "turn off automatic margins"), + TermCapability(:xon_character, :xonc, "XON character"), + TermCapability(:xoff_character, :xoffc, "XOFF character"), + TermCapability(:ena_acs, :enacs, "enable alternate char set"), + TermCapability(:label_on, :smln, "turn on soft labels"), + TermCapability(:label_off, :rmln, "turn off soft labels"), + TermCapability(:key_beg, :kbeg, "begin key"), + TermCapability(:key_cancel, :kcan, "cancel key"), + TermCapability(:key_close, :kclo, "close key"), + TermCapability(:key_command, :kcmd, "command key"), + TermCapability(:key_copy, :kcpy, "copy key"), + TermCapability(:key_create, :kcrt, "create key"), + TermCapability(:key_end, :kend, "end key"), + TermCapability(:key_enter, :kent, "enter/send key"), + TermCapability(:key_exit, :kext, "exit key"), + TermCapability(:key_find, :kfnd, "find key"), + TermCapability(:key_help, :khlp, "help key"), + TermCapability(:key_mark, :kmrk, "mark key"), + TermCapability(:key_message, :kmsg, "message key"), + TermCapability(:key_move, :kmov, "move key"), + TermCapability(:key_next, :knxt, "next key"), + TermCapability(:key_open, :kopn, "open key"), + TermCapability(:key_options, :kopt, "options key"), + TermCapability(:key_previous, :kprv, "previous key"), + TermCapability(:key_print, :kprt, "print key"), + TermCapability(:key_redo, :krdo, "redo key"), + TermCapability(:key_reference, :kref, "reference key"), + TermCapability(:key_refresh, :krfr, "refresh key"), + TermCapability(:key_replace, :krpl, "replace key"), + TermCapability(:key_restart, :krst, "restart key"), + TermCapability(:key_resume, :kres, "resume key"), + TermCapability(:key_save, :ksav, "save key"), + TermCapability(:key_suspend, :kspd, "suspend key"), + TermCapability(:key_undo, :kund, "undo key"), + TermCapability(:key_sbeg, :kBEG, "shifted begin key"), + TermCapability(:key_scancel, :kCAN, "shifted cancel key"), + TermCapability(:key_scommand, :kCMD, "shifted command key"), + TermCapability(:key_scopy, :kCPY, "shifted copy key"), + TermCapability(:key_screate, :kCRT, "shifted create key"), + TermCapability(:key_sdc, :kDC, "shifted delete-character key"), + TermCapability(:key_sdl, :kDL, "shifted delete-line key"), + TermCapability(:key_select, :kslt, "select key"), + TermCapability(:key_send, :kEND, "shifted end key"), + TermCapability(:key_seol, :kEOL, "shifted clear-to-end-of-line key"), + TermCapability(:key_sexit, :kEXT, "shifted exit key"), + TermCapability(:key_sfind, :kFND, "shifted find key"), + TermCapability(:key_shelp, :kHLP, "shifted help key"), + TermCapability(:key_shome, :kHOM, "shifted home key"), + TermCapability(:key_sic, :kIC, "shifted insert-character key"), + TermCapability(:key_sleft, :kLFT, "shifted left-arrow key"), + TermCapability(:key_smessage, :kMSG, "shifted message key"), + TermCapability(:key_smove, :kMOV, "shifted move key"), + TermCapability(:key_snext, :kNXT, "shifted next key"), + TermCapability(:key_soptions, :kOPT, "shifted options key"), + TermCapability(:key_sprevious, :kPRV, "shifted previous key"), + TermCapability(:key_sprint, :kPRT, "shifted print key"), + TermCapability(:key_sredo, :kRDO, "shifted redo key"), + TermCapability(:key_sreplace, :kRPL, "shifted replace key"), + TermCapability(:key_sright, :kRIT, "shifted right-arrow key"), + TermCapability(:key_srsume, :kRES, "shifted resume key"), + TermCapability(:key_ssave, :kSAV, "shifted save key"), + TermCapability(:key_ssuspend, :kSPD, "shifted suspend key"), + TermCapability(:key_sundo, :kUND, "shifted undo key"), + TermCapability(:req_for_input, :rfi, "send next input char (for ptys)"), + TermCapability(:key_f11, :kf11, "F11 function key"), + TermCapability(:key_f12, :kf12, "F12 function key"), + TermCapability(:key_f13, :kf13, "F13 function key"), + TermCapability(:key_f14, :kf14, "F14 function key"), + TermCapability(:key_f15, :kf15, "F15 function key"), + TermCapability(:key_f16, :kf16, "F16 function key"), + TermCapability(:key_f17, :kf17, "F17 function key"), + TermCapability(:key_f18, :kf18, "F18 function key"), + TermCapability(:key_f19, :kf19, "F19 function key"), + TermCapability(:key_f20, :kf20, "F20 function key"), + TermCapability(:key_f21, :kf21, "F21 function key"), + TermCapability(:key_f22, :kf22, "F22 function key"), + TermCapability(:key_f23, :kf23, "F23 function key"), + TermCapability(:key_f24, :kf24, "F24 function key"), + TermCapability(:key_f25, :kf25, "F25 function key"), + TermCapability(:key_f26, :kf26, "F26 function key"), + TermCapability(:key_f27, :kf27, "F27 function key"), + TermCapability(:key_f28, :kf28, "F28 function key"), + TermCapability(:key_f29, :kf29, "F29 function key"), + TermCapability(:key_f30, :kf30, "F30 function key"), + TermCapability(:key_f31, :kf31, "F31 function key"), + TermCapability(:key_f32, :kf32, "F32 function key"), + TermCapability(:key_f33, :kf33, "F33 function key"), + TermCapability(:key_f34, :kf34, "F34 function key"), + TermCapability(:key_f35, :kf35, "F35 function key"), + TermCapability(:key_f36, :kf36, "F36 function key"), + TermCapability(:key_f37, :kf37, "F37 function key"), + TermCapability(:key_f38, :kf38, "F38 function key"), + TermCapability(:key_f39, :kf39, "F39 function key"), + TermCapability(:key_f40, :kf40, "F40 function key"), + TermCapability(:key_f41, :kf41, "F41 function key"), + TermCapability(:key_f42, :kf42, "F42 function key"), + TermCapability(:key_f43, :kf43, "F43 function key"), + TermCapability(:key_f44, :kf44, "F44 function key"), + TermCapability(:key_f45, :kf45, "F45 function key"), + TermCapability(:key_f46, :kf46, "F46 function key"), + TermCapability(:key_f47, :kf47, "F47 function key"), + TermCapability(:key_f48, :kf48, "F48 function key"), + TermCapability(:key_f49, :kf49, "F49 function key"), + TermCapability(:key_f50, :kf50, "F50 function key"), + TermCapability(:key_f51, :kf51, "F51 function key"), + TermCapability(:key_f52, :kf52, "F52 function key"), + TermCapability(:key_f53, :kf53, "F53 function key"), + TermCapability(:key_f54, :kf54, "F54 function key"), + TermCapability(:key_f55, :kf55, "F55 function key"), + TermCapability(:key_f56, :kf56, "F56 function key"), + TermCapability(:key_f57, :kf57, "F57 function key"), + TermCapability(:key_f58, :kf58, "F58 function key"), + TermCapability(:key_f59, :kf59, "F59 function key"), + TermCapability(:key_f60, :kf60, "F60 function key"), + TermCapability(:key_f61, :kf61, "F61 function key"), + TermCapability(:key_f62, :kf62, "F62 function key"), + TermCapability(:key_f63, :kf63, "F63 function key"), + TermCapability(:clr_bol, :el1, "Clear to beginning of line"), + TermCapability(:clear_margins, :mgc, "clear right and left soft margins"), + TermCapability(:set_left_margin, :smgl, "set left soft margin at current column."), + TermCapability(:set_right_margin, :smgr, "set right soft margin at current column"), + TermCapability(:label_format, :fln, "label format"), + TermCapability(:set_clock, :sclk, "set clock, #1 hrs #2 mins #3 secs"), + TermCapability(:display_clock, :dclk, "display clock"), + TermCapability(:remove_clock, :rmclk, "remove clock"), + TermCapability(:create_window, :cwin, "define a window #1 from #2,#3 to #4,#5"), + TermCapability(:goto_window, :wingo, "go to window #1"), + TermCapability(:hangup, :hup, "hang-up phone"), + TermCapability(:dial_phone, :dial, "dial number #1"), + TermCapability(:quick_dial, :qdial, "dial number #1 without checking"), + TermCapability(:tone, :tone, "select touch tone dialing"), + TermCapability(:pulse, :pulse, "select pulse dialing"), + TermCapability(:flash_hook, :hook, "flash switch hook"), + TermCapability(:fixed_pause, :pause, "pause for 2-3 seconds"), + TermCapability(:wait_tone, :wait, "wait for dial-tone"), + TermCapability(:user0, :u0, "User string #0"), + TermCapability(:user1, :u1, "User string #1"), + TermCapability(:user2, :u2, "User string #2"), + TermCapability(:user3, :u3, "User string #3"), + TermCapability(:user4, :u4, "User string #4"), + TermCapability(:user5, :u5, "User string #5"), + TermCapability(:user6, :u6, "User string #6"), + TermCapability(:user7, :u7, "User string #7"), + TermCapability(:user8, :u8, "User string #8"), + TermCapability(:user9, :u9, "User string #9"), + TermCapability(:orig_pair, :op, "Set default pair to its original value"), + TermCapability(:orig_colors, :oc, "Set all color pairs to the original ones"), + TermCapability(:initialize_color, :initc, "initialize color #1 to (#2,#3,#4)"), + TermCapability(:initialize_pair, :initp, "Initialize color pair #1 to fg=(#2,#3,#4), bg=(#5,#6,#7)"), + TermCapability(:set_color_pair, :scp, "Set current color pair to #1"), + TermCapability(:set_foreground, :setf, "Set foreground color #1"), + TermCapability(:set_background, :setb, "Set background color #1"), + TermCapability(:change_char_pitch, :cpi, "Change number of characters per inch to #1"), + TermCapability(:change_line_pitch, :lpi, "Change number of lines per inch to #1"), + TermCapability(:change_res_horz, :chr, "Change horizontal resolution to #1"), + TermCapability(:change_res_vert, :cvr, "Change vertical resolution to #1"), + TermCapability(:define_char, :defc, "Define a character #1, #2 dots wide, descender #3"), + TermCapability(:enter_doublewide_mode, :swidm, "Enter double-wide mode"), + TermCapability(:enter_draft_quality, :sdrfq, "Enter draft-quality mode"), + TermCapability(:enter_italics_mode, :sitm, "Enter italic mode"), + TermCapability(:enter_leftward_mode, :slm, "Start leftward carriage motion"), + TermCapability(:enter_micro_mode, :smicm, "Start micro-motion mode"), + TermCapability(:enter_near_letter_quality, :snlq, "Enter NLQ mode"), + TermCapability(:enter_normal_quality, :snrmq, "Enter normal-quality mode"), + TermCapability(:enter_shadow_mode, :sshm, "Enter shadow-print mode"), + TermCapability(:enter_subscript_mode, :ssubm, "Enter subscript mode"), + TermCapability(:enter_superscript_mode, :ssupm, "Enter superscript mode"), + TermCapability(:enter_upward_mode, :sum, "Start upward carriage motion"), + TermCapability(:exit_doublewide_mode, :rwidm, "End double-wide mode"), + TermCapability(:exit_italics_mode, :ritm, "End italic mode"), + TermCapability(:exit_leftward_mode, :rlm, "End left-motion mode"), + TermCapability(:exit_micro_mode, :rmicm, "End micro-motion mode"), + TermCapability(:exit_shadow_mode, :rshm, "End shadow-print mode"), + TermCapability(:exit_subscript_mode, :rsubm, "End subscript mode"), + TermCapability(:exit_superscript_mode, :rsupm, "End superscript mode"), + TermCapability(:exit_upward_mode, :rum, "End reverse character motion"), + TermCapability(:micro_column_address, :mhpa, "Like column_address in micro mode"), + TermCapability(:micro_down, :mcud1, "Like cursor_down in micro mode"), + TermCapability(:micro_left, :mcub1, "Like cursor_left in micro mode"), + TermCapability(:micro_right, :mcuf1, "Like cursor_right in micro mode"), + TermCapability(:micro_row_address, :mvpa, "Like row_address #1 in micro mode"), + TermCapability(:micro_up, :mcuu1, "Like cursor_up in micro mode"), + TermCapability(:order_of_pins, :porder, "Match software bits to print-head pins"), + TermCapability(:parm_down_micro, :mcud, "Like parm_down_cursor in micro mode"), + TermCapability(:parm_left_micro, :mcub, "Like parm_left_cursor in micro mode"), + TermCapability(:parm_right_micro, :mcuf, "Like parm_right_cursor in micro mode"), + TermCapability(:parm_up_micro, :mcuu, "Like parm_up_cursor in micro mode"), + TermCapability(:select_char_set, :scs, "Select character set, #1"), + TermCapability(:set_bottom_margin, :smgb, "Set bottom margin at current line"), + TermCapability(:set_bottom_margin_parm, :smgbp, "Set bottom margin at line #1 or (if smgtp is not given) #2 lines from bottom"), + TermCapability(:set_left_margin_parm, :smglp, "Set left (right) margin at column #1"), + TermCapability(:set_right_margin_parm, :smgrp, "Set right margin at column #1"), + TermCapability(:set_top_margin, :smgt, "Set top margin at current line"), + TermCapability(:set_top_margin_parm, :smgtp, "Set top (bottom) margin at row #1"), + TermCapability(:start_bit_image, :sbim, "Start printing bit image graphics"), + TermCapability(:start_char_set_def, :scsd, "Start character set definition #1, with #2 characters in the set"), + TermCapability(:stop_bit_image, :rbim, "Stop printing bit image graphics"), + TermCapability(:stop_char_set_def, :rcsd, "End definition of character set #1"), + TermCapability(:subscript_characters, :subcs, "List of subscriptable characters"), + TermCapability(:superscript_characters, :supcs, "List of superscriptable characters"), + TermCapability(:these_cause_cr, :docr, "Printing any of these characters causes CR"), + TermCapability(:zero_motion, :zerom, "No motion for subsequent character"), + TermCapability(:char_set_names, :csnm, "Produce #1'th item from list of character set names"), + TermCapability(:key_mouse, :kmous, "Mouse event has occurred"), + TermCapability(:mouse_info, :minfo, "Mouse status information"), + TermCapability(:req_mouse_pos, :reqmp, "Request mouse position"), + TermCapability(:get_mouse, :getm, "Curses should get button events, parameter #1 not documented."), + TermCapability(:set_a_foreground, :setaf, "Set foreground color to #1, using ANSI escape"), + TermCapability(:set_a_background, :setab, "Set background color to #1, using ANSI escape"), + TermCapability(:pkey_plab, :pfxl, "Program function key #1 to type string #2 and show string #3"), + TermCapability(:device_type, :devt, "Indicate language/codeset support"), + TermCapability(:code_set_init, :csin, "Init sequence for multiple codesets"), + TermCapability(:set0_des_seq, :s0ds, "Shift to codeset 0 (EUC set 0, ASCII)"), + TermCapability(:set1_des_seq, :s1ds, "Shift to codeset 1"), + TermCapability(:set2_des_seq, :s2ds, "Shift to codeset 2"), + TermCapability(:set3_des_seq, :s3ds, "Shift to codeset 3"), + TermCapability(:set_lr_margin, :smglr, "Set both left and right margins to #1, #2. (ML is not in BSD termcap)."), + TermCapability(:set_tb_margin, :smgtb, "Sets both top and bottom margins to #1, #2"), + TermCapability(:bit_image_repeat, :birep, "Repeat bit image cell #1 #2 times"), + TermCapability(:bit_image_newline, :binel, "Move to next row of the bit image"), + TermCapability(:bit_image_carriage_return, :bicr, "Move to beginning of same row"), + TermCapability(:color_names, :colornm, "Give name for color #1"), + TermCapability(:define_bit_image_region, :defbi, "Define rectangular bit image region"), + TermCapability(:end_bit_image_region, :endbi, "End a bit-image region"), + TermCapability(:set_color_band, :setcolor, "Change to ribbon color #1"), + TermCapability(:set_page_length, :slines, "Set page length to #1 lines"), + TermCapability(:display_pc_char, :dispc, "Display PC character #1"), + TermCapability(:enter_pc_charset_mode, :smpch, "Enter PC character display mode"), + TermCapability(:exit_pc_charset_mode, :rmpch, "Exit PC character display mode"), + TermCapability(:enter_scancode_mode, :smsc, "Enter PC scancode mode"), + TermCapability(:exit_scancode_mode, :rmsc, "Exit PC scancode mode"), + TermCapability(:pc_term_options, :pctrm, "PC terminal options"), + TermCapability(:scancode_escape, :scesc, "Escape for scancode emulation"), + TermCapability(:alt_scancode_esc, :scesa, "Alternate escape for scancode emulation"), + TermCapability(:enter_horizontal_hl_mode, :ehhlm, "Enter horizontal highlight mode"), + TermCapability(:enter_left_hl_mode, :elhlm, "Enter left highlight mode"), + TermCapability(:enter_low_hl_mode, :elohlm, "Enter low highlight mode"), + TermCapability(:enter_right_hl_mode, :erhlm, "Enter right highlight mode"), + TermCapability(:enter_top_hl_mode, :ethlm, "Enter top highlight mode"), + TermCapability(:enter_vertical_hl_mode, :evhlm, "Enter vertical highlight mode"), + TermCapability(:set_a_attributes, :sgr1, "Define second set of video attributes #1-#6"), + TermCapability(:set_pglen_inch, :slength, "Set page length to #1 hundredth of an inch (some implementations use sL for termcap)."), + TermCapability(:termcap_init2, :OTi2, "secondary initialization string"), + TermCapability(:termcap_reset, :OTrs, "terminal reset string"), + TermCapability(:linefeed_if_not_lf, :OTnl, "use to move down"), + TermCapability(:backspace_if_not_bs, :OTbc, "move left, if not ^H"), + TermCapability(:other_non_function_keys, :OTko, "list of self-mapped keycaps"), + TermCapability(:arrow_key_map, :OTma, "map motion-keys for vi version 2"), + TermCapability(:acs_ulcorner, :OTG2, "single upper left"), + TermCapability(:acs_llcorner, :OTG3, "single lower left"), + TermCapability(:acs_urcorner, :OTG1, "single upper right"), + TermCapability(:acs_lrcorner, :OTG4, "single lower right"), + TermCapability(:acs_ltee, :OTGR, "tee pointing right"), + TermCapability(:acs_rtee, :OTGL, "tee pointing left"), + TermCapability(:acs_btee, :OTGU, "tee pointing up"), + TermCapability(:acs_ttee, :OTGD, "tee pointing down"), + TermCapability(:acs_hline, :OTGH, "single horizontal line"), + TermCapability(:acs_vline, :OTGV, "single vertical line"), + TermCapability(:acs_plus, :OTGC, "single intersection"), + TermCapability(:memory_lock, :meml, "lock memory above cursor"), + TermCapability(:memory_unlock, :memu, "unlock memory"), + TermCapability(:box_chars_1, :box1, "box characters primary set"), ] + +""" +Terminfo extensions that NCurses 6.4-20230311 is aware of. +""" +const TERM_USER = Dict{Tuple{DataType, Symbol}, Union{Tuple{Nothing, String}, Tuple{Symbol, String}}}( + (Int, :CO ) => (nothing, "number of indexed colors overlaying RGB space"), + (String, :E3) => (nothing, "clears the terminal's scrollback buffer."), + (Bool, :NQ) => (nothing, "terminal does not support query/response"), + (Bool, :RGB) => (nothing, "use direct colors with 1/3 of color-pair bits per color."), + (Int, :RGB) => (nothing, "use direct colors with given number of bits per color."), + (String, :RGB) => (nothing, "use direct colors with given bit-layout."), + (String, :TS) => (nothing, "like \"tsl\", but uses no parameter."), + (Int, :U8) => (nothing, "terminal does/does not support VT100 SI/SO when processing UTF-8 encoding."), + (String, :XM) => (nothing, "initialize alternate xterm mouse mode"), + (String, :grbom) => (nothing, "disable real bold (not intensity bright) mode."), + (String, :gsbom) => (nothing, "enable real bold (not intensity bright) mode."), + (String, :xm) => (nothing, "mouse response"), + (String, :Rmol) => (:exit_overline_mode, "remove overline-mode"), + (String, :Smol) => (:enter_overline_mode, "set overline-mode"), + (String, :blink2) => (nothing, "turn on rapid blinking"), + (String, :norm) => (nothing, "turn off bold and half-bright mode"), + (String, :opaq) => (nothing, "turn off blank mode"), + (String, :setal) => (nothing, "set underline-color"), + (String, :smul2) => (nothing, "begin double underline mode"), + (Bool, :AN) => (nothing, "turn on autonuke."), + (Bool, :AX) => (nothing, "understands ANSI set default fg/bg color (\\E[39m / \\E[49m)."), + (String, :C0) => (nothing, "use the string as a conversion table for font '0', like acsc."), + (Bool, :C8) => (nothing, "terminal shows bold as high-intensity colors."), + (String, :CE) => (nothing, "switch cursor-keys back to normal mode."), + (String, :CS) => (nothing, "switch cursor-keys to application mode."), + (String, :E0) => (nothing, "switch charset 'G0' back to standard charset. Default is '\\E(B'."), + (Bool, :G0) => (nothing, "terminal can deal with ISO 2022 font selection sequences."), + (String, :KJ) => (nothing, "set the encoding of the terminal."), + (Int, :OL) => (nothing, "set the screen program's output buffer limit."), + (String, :S0) => (nothing, "switch charset 'G0' to the specified charset. Default is '\\E(%.'."), + (Bool, :TF) => (nothing, "add missing capabilities to screen's termcap/info entry. (Set by default)."), + (String, :WS) => (nothing, "resize display. This capability has the desired width and height as arguments. SunView(tm) example: '\\E[8;%d;%dt'."), + (String, :XC) => (nothing, "describe a translation of characters to strings depending on the current font."), + (Bool, :XT) => (nothing, "terminal understands special xterm sequences (OSC, mouse tracking)."), + (String, :Z0) => (nothing, "change width to 132 columns."), + (String, :Z1) => (nothing, "change width to 80 columns."), + (String, :Cr) => (:reset_cursor_color, "restore the default cursor color."), + (String, :Cs) => (:set_cursor_color, "set the cursor color."), + (String, :Csr) => (nothing, "change the cursor style, overriding Ss."), + (String, :Ms) => (:set_host_clipboard, "store the current buffer in the host terminal's selection (clipboard)."), + (String, :Se) => (:reset_cursor_style, "reset the cursor style to the terminal initial state."), + (String, :Smulx) => (:set_underline_style, "modify the appearance of underlines in VTE."), + (String, :Ss) => (:set_cursor_style, "change the cursor style."), + (String, :rmxx) => (:exit_strikeout_mode, "reset ECMA-48 strikeout/crossed-out attributes."), + (String, :smxx) => (:enter_strikeout_mode, "set ECMA-48 strikeout/crossed-out attributes."), + (String, :BD) => (nothing, "disables bracketed paste"), + (String, :BE) => (nothing, "enables bracketed paste"), + (String, :PE) => (nothing, "is sent after pasted text"), + (String, :PS) => (nothing, "is sent before pasted text"), + (String, :RV) => (nothing, "report terminal secondary device attributes"), + (String, :XR) => (nothing, "report terminal version as a free-format string."), + (Bool, :XF) => (:xterm_focus, "terminal supports xterm focus in/out"), + (String, :rv) => (nothing, "response to RV, regular expression"), + (String, :xr) => (nothing, "response to XR, regular expression"), + (String, :csl) => (:clear_status_line, "clear status line"), + (String, :kDC3) => (:key_alt_delete_character, "alt delete-character"), + (String, :kDC4) => (:key_shift_alt_delete_character, "shift+alt delete-character"), + (String, :kDC5) => (:key_control_delete_character, "control delete-character"), + (String, :kDC6) => (:key_shift_control_delete_character, "shift+control delete-character"), + (String, :kDC7) => (:key_alt_control_delete_character, "alt+control delete-character"), + (String, :kDN) => (:key_shift_down_cursor, "shift down-cursor"), + (String, :kDN3) => (:key_alt_down_cursor, "alt down-cursor"), + (String, :kDN4) => (:key_shift_alt_down_cursor, "shift+alt down-cursor"), + (String, :kDN5) => (:key_control_down_cursor, "control down-cursor"), + (String, :kDN6) => (:key_shift_control_down_cursor, "shift+control down-cursor"), + (String, :kDN7) => (:key_alt_control_down_cursor, "alt+control down-cursor"), + (String, :kEND3) => (:key_alt_end, "alt end"), + (String, :kEND4) => (:key_shift_alt_end, "shift+alt end"), + (String, :kEND5) => (:key_control_end, "control end"), + (String, :kEND6) => (:key_shift_control_end, "shift+control end"), + (String, :kEND7) => (:key_alt_control_end, "alt+control end"), + (String, :kHOM3) => (:key_alt_home, "alt home"), + (String, :kHOM4) => (:key_shift_alt_home, "shift+alt home"), + (String, :kHOM5) => (:key_control_home, "control home"), + (String, :kHOM6) => (:key_shift_control_home, "shift+control home"), + (String, :kHOM7) => (:key_alt_control_home, "alt+control home"), + (String, :kIC3) => (:key_alt_insert_character, "alt insert-character"), + (String, :kIC4) => (:key_shift_alt_insert_character, "shift+alt insert-character"), + (String, :kIC5) => (:key_control_insert_character, "control insert-character"), + (String, :kIC6) => (:key_shift_control_insert_character, "shift+control insert-character"), + (String, :kIC7) => (:key_alt_control_insert_character, "alt+control insert-character"), + (String, :kLFT3) => (:key_alt_left_cursor, "alt left-cursor"), + (String, :kLFT4) => (:key_shift_alt_left_cursor, "shift+alt left-cursor"), + (String, :kLFT5) => (:key_control_left_cursor, "control left-cursor"), + (String, :kLFT6) => (:key_shift_control_left_cursor, "shift+control left-cursor"), + (String, :kLFT7) => (:key_alt_control_left_cursor, "alt+control left-cursor"), + (String, :kNXT3) => (:key_alt_next, "alt next"), + (String, :kNXT4) => (:key_shift_alt_next, "shift+alt next"), + (String, :kNXT5) => (:key_control_next, "control next"), + (String, :kNXT6) => (:key_shift_control_next, "shift+control next"), + (String, :kNXT7) => (:key_alt_control_next, "alt+control next"), + (String, :kPRV3) => (:key_alt_previous, "alt previous"), + (String, :kPRV4) => (:key_shift_alt_previous, "shift+alt previous"), + (String, :kPRV5) => (:key_control_previous, "control previous"), + (String, :kPRV6) => (:key_shift_control_previous, "shift+control previous"), + (String, :kPRV7) => (:key_alt_control_previous, "alt+control previous"), + (String, :kRIT3) => (:key_alt_right_cursor, "alt right-cursor"), + (String, :kRIT4) => (:key_shift_alt_right_cursor, "shift+alt right-cursor"), + (String, :kRIT5) => (:key_control_right_cursor, "control right-cursor"), + (String, :kRIT6) => (:key_shift_control_right_cursor, "shift+control right-cursor"), + (String, :kRIT7) => (:key_alt_control_right_cursor, "alt+control right-cursor"), + (String, :kUP) => (:key_shift_up_cursor, "shift up-cursor"), + (String, :kUP3) => (:key_alt_up_cursor, "alt up-cursor"), + (String, :kUP4) => (:key_shift_alt_up_cursor, "shift+alt up-cursor"), + (String, :kUP5) => (:key_control_up_cursor, "control up-cursor"), + (String, :kUP6) => (:key_shift_control_up_cursor, "shift+control up-cursor"), + (String, :kUP7) => (:key_alt_control_up_cursor, "alt+control up-cursor"), + (String, :ka2) => (nothing, "vt220-keypad extensions"), + (String, :kb1) => (nothing, "vt220-keypad extensions"), + (String, :kb3) => (nothing, "vt220-keypad extensions"), + (String, :kc2) => (nothing, "vt220-keypad extensions"), + (String, :kxIN) => (:key_mouse_response_on_focus_in, "mouse response on focus-in"), + (String, :kxOUT) => (:key_mouse_response_on_focus_out, "mouse response on focus-out"), + (Bool, :Tc) => (:truecolor, "tmux extension to indicate 24-bit truecolor support"), + (Bool, :Su) => (:can_style_underline, "kitty extension to indicate styled underline support"), +) diff --git a/test/terminfo.jl b/test/terminfo.jl index 809b0374ad3e1..07aa21704fef5 100644 --- a/test/terminfo.jl +++ b/test/terminfo.jl @@ -356,14 +356,34 @@ let 0x73, 0x6d, 0x78, 0x78, 0x00, 0x78, 0x6d, 0x00] xterm_extensions = - [:kEND5, :Cs, :kDN5, :Cr, :kDC6, :kPRV6, :kDN7, :kb1, :kpZRO, :kNXT6, - :kLFT5, :kPRV3, :kRIT4, :kDC4, :kc2, :kp5, :kLFT6, :kIC6, :kEND6, :kIC4, - :kRIT7, :rmxx, :kpADD, :xm, :kNXT3, :XT, :kIC7, :kHOM4, :kDC7, :kPRV7, - :ka2, :kUP7, :kDN6, :kIC5, :kNXT4, :kUP5, :AX, :kpSUB, :kb3, :kDN4, - :kHOM5, :kHOM6, :kDN3, :kLFT4, :kRIT5, :kIC3, :kPRV4, :kUP, :kRIT6, :E3, - :kEND3, :kHOM7, :kDC3, :kLFT7, :kNXT5, :Se, :Ss, :kHOM3, :kRIT3, :kNXT7, - :smxx, :kEND4, :kDN, :kUP6, :XM, :kPRV5, :kUP4, :kpDOT, :kpMUL, :kEND7, - :Ms, :kpCMA, :kDC5, :kLFT3, :kpDIV, :kUP3] + [:AX, :E3, :XM, :XT, :enter_strikeout_mode, :exit_strikeout_mode, :ka2, + :kb1, :kb3, :kc2, :key_alt_control_delete_character, + :key_alt_control_down_cursor, :key_alt_control_end, + :key_alt_control_home, :key_alt_control_insert_character, + :key_alt_control_left_cursor, :key_alt_control_next, + :key_alt_control_previous, :key_alt_control_right_cursor, + :key_alt_control_up_cursor, :key_alt_delete_character, + :key_alt_down_cursor, :key_alt_end, :key_alt_home, + :key_alt_insert_character, :key_alt_left_cursor, :key_alt_next, + :key_alt_previous, :key_alt_right_cursor, :key_alt_up_cursor, + :key_control_delete_character, :key_control_down_cursor, + :key_control_end, :key_control_home, :key_control_insert_character, + :key_control_left_cursor, :key_control_next, :key_control_previous, + :key_control_right_cursor, :key_control_up_cursor, + :key_shift_alt_delete_character, :key_shift_alt_down_cursor, + :key_shift_alt_end, :key_shift_alt_home, + :key_shift_alt_insert_character, :key_shift_alt_left_cursor, + :key_shift_alt_next, :key_shift_alt_previous, + :key_shift_alt_right_cursor, :key_shift_alt_up_cursor, + :key_shift_control_delete_character, :key_shift_control_down_cursor, + :key_shift_control_end, :key_shift_control_home, + :key_shift_control_insert_character, :key_shift_control_left_cursor, + :key_shift_control_next, :key_shift_control_previous, + :key_shift_control_right_cursor, :key_shift_control_up_cursor, + :key_shift_down_cursor, :key_shift_up_cursor, :kp5, :kpADD, :kpCMA, + :kpDIV, :kpDOT, :kpMUL, :kpSUB, :kpZRO, :reset_cursor_color, + :reset_cursor_style, :set_cursor_color, :set_cursor_style, + :set_host_clipboard, :xm] xterm_capabilities = Dict{Symbol, Union{Bool, Int, String}}( :AX => true, @@ -889,9 +909,9 @@ let @testset "terminfo" begin dumb = Base.TermInfo(read(IOBuffer(dumb_terminfo), Base.TermInfoRaw)) @test dumb.names == ["dumb", "80-column dumb tty"] - @test length(dumb.flags) == 4 - @test length(dumb.numbers) == 2 - @test length(dumb.strings) == 8 + @test length(dumb.flags) == 2 + @test length(dumb.numbers) == 1 + @test length(dumb.strings) == 4 @test isnothing(dumb.extensions) for (key, value) in dumb_capabilities @test dumb[key] == value @@ -899,9 +919,9 @@ let xterm = Base.TermInfo(read(IOBuffer(xterm_terminfo), Base.TermInfoRaw)) @test xterm.names == ["xterm", "xterm terminal emulator (X Window System)"] - @test length(xterm.flags) == 78 - @test length(xterm.numbers) == 29 - @test length(xterm.strings) == 432 + @test length(xterm.flags) == 40 + @test length(xterm.numbers) == 15 + @test length(xterm.strings) == 253 @test sort(xterm.extensions |> collect) == sort(xterm_extensions) for (key, value) in xterm_capabilities @test xterm[key] == value From 0f0f8eb8fa73b622ea04525567f4b26237bc8d39 Mon Sep 17 00:00:00 2001 From: Shuhei Kadowaki <40514306+aviatesk@users.noreply.github.com> Date: Tue, 27 Feb 2024 15:12:46 +0900 Subject: [PATCH 23/25] minor fixes on test/precompile.jl (#53476) These changes are driven-by fixes I found during investigating into a more complex issue related to precompilation with external abs int. --- test/precompile.jl | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/test/precompile.jl b/test/precompile.jl index 344b98b2ee186..527be616a5448 100644 --- a/test/precompile.jl +++ b/test/precompile.jl @@ -1723,7 +1723,6 @@ let newinterp_path = abspath("compiler/newinterp.jl") import SimpleModule: basic_caller, basic_callee module Custom - const CC = Core.Compiler include("$($newinterp_path)") @newinterp PrecompileInterpreter end @@ -1822,7 +1821,7 @@ let newinterp_path = abspath("compiler/newinterp.jl") using CustomAbstractInterpreterCaching2 cache_owner = Core.Compiler.cache_owner( CustomAbstractInterpreterCaching2.Custom.PrecompileInterpreter()) - let m = only(methods(CustomAbstractInterpreterCaching.basic_callee)) + let m = only(methods(CustomAbstractInterpreterCaching2.basic_callee)) mi = only(Base.specializations(m)) ci = mi.cache @test isdefined(ci, :next) From 7ef180e4c05c68ea44d7b044fd68efcd4dea70fb Mon Sep 17 00:00:00 2001 From: Ian Butterworth Date: Thu, 22 Feb 2024 04:30:07 -0500 Subject: [PATCH 24/25] protect against PkgId and UUID being imported and losing Base prefix in create_expr_cache (#53387) Fixes https://github.com/JuliaLang/julia/issues/53381 (cherry picked from commit a1db8daafd6c1a1179de77955167a6958a4c23b0) --- base/loading.jl | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/base/loading.jl b/base/loading.jl index 0ec9342582db5..2be97237238dd 100644 --- a/base/loading.jl +++ b/base/loading.jl @@ -2701,6 +2701,7 @@ function create_expr_cache(pkg::PkgId, input::String, output::String, output_o:: error("LOAD_PATH entries cannot contain $(repr(path_sep))") deps_strs = String[] + # protects against PkgId and UUID being imported and losing Base prefix function pkg_str(_pkg::PkgId) if _pkg.uuid === nothing "Base.PkgId($(repr(_pkg.name)))" @@ -2711,6 +2712,9 @@ function create_expr_cache(pkg::PkgId, input::String, output::String, output_o:: for (pkg, build_id) in concrete_deps push!(deps_strs, "$(pkg_str(pkg)) => $(repr(build_id))") end + deps_eltype = sprint(show, eltype(concrete_deps); context = :module=>nothing) + deps = deps_eltype * "[" * join(deps_strs, ",") * "]" + precomp_stack = "Base.PkgId[$(join(map(pkg_str, vcat(Base.precompilation_stack, pkg)), ", "))]" if output_o !== nothing @debug "Generating object cache file for $(repr("text/plain", pkg))" @@ -2722,8 +2726,6 @@ function create_expr_cache(pkg::PkgId, input::String, output::String, output_o:: opts = `-O0 --output-ji $(output) --output-incremental=yes` end - deps_eltype = sprint(show, eltype(concrete_deps); context = :module=>nothing) - deps = deps_eltype * "[" * join(deps_strs, ",") * "]" trace = isassigned(PRECOMPILE_TRACE_COMPILE) ? `--trace-compile=$(PRECOMPILE_TRACE_COMPILE[])` : `` io = open(pipeline(addenv(`$(julia_cmd(;cpu_target)::Cmd) $(flags) @@ -2739,7 +2741,7 @@ function create_expr_cache(pkg::PkgId, input::String, output::String, output_o:: # write data over stdin to avoid the (unlikely) case of exceeding max command line size write(io.in, """ empty!(Base.EXT_DORMITORY) # If we have a custom sysimage with `EXT_DORMITORY` prepopulated - Base.track_nested_precomp($(vcat(Base.precompilation_stack, pkg))) + Base.track_nested_precomp($precomp_stack) Base.precompiling_extension = $(loading_extension) Base.include_package_for_output($(pkg_str(pkg)), $(repr(abspath(input))), $(repr(depot_path)), $(repr(dl_load_path)), $(repr(load_path)), $deps, $(repr(source_path(nothing)))) From d6bbf85398d0bb5bf14b510fb13f08d83730b3ee Mon Sep 17 00:00:00 2001 From: Shuhei Kadowaki <40514306+aviatesk@users.noreply.github.com> Date: Fri, 1 Mar 2024 09:43:00 +0900 Subject: [PATCH 25/25] post-opt analysis: fix conditional successor visitation logic (#53518) Previously `conditional_successors_may_throw` performed post-domination analysis not on the initially specified `bb` (which was given as the argument), but on those BBs being analyzed that were popped from the work-queue at the time. As a result, there were cases where not all conditional successors were visited. fixes #53508 --- base/compiler/optimize.jl | 21 ++++++++++++++------- test/compiler/effects.jl | 5 +++++ test/compiler/ssair.jl | 29 +++++++++++++++++++++++++++++ 3 files changed, 48 insertions(+), 7 deletions(-) diff --git a/base/compiler/optimize.jl b/base/compiler/optimize.jl index cbe80a65fc607..f6d98fa53dede 100644 --- a/base/compiler/optimize.jl +++ b/base/compiler/optimize.jl @@ -534,17 +534,22 @@ function any_stmt_may_throw(ir::IRCode, bb::Int) return false end -function conditional_successors_may_throw(lazypostdomtree::LazyPostDomtree, ir::IRCode, bb::Int) +function visit_conditional_successors(callback, lazypostdomtree::LazyPostDomtree, ir::IRCode, bb::Int) visited = BitSet((bb,)) worklist = Int[bb] while !isempty(worklist) - thisbb = pop!(worklist) + thisbb = popfirst!(worklist) for succ in ir.cfg.blocks[thisbb].succs succ in visited && continue push!(visited, succ) - postdominates(get!(lazypostdomtree), succ, thisbb) && continue - any_stmt_may_throw(ir, succ) && return true - push!(worklist, succ) + if postdominates(get!(lazypostdomtree), succ, bb) + # this successor is not conditional, so no need to visit it further + continue + elseif callback(succ) + return true + else + push!(worklist, succ) + end end end return false @@ -836,8 +841,10 @@ function ((; sv)::ScanStmt)(inst::Instruction, lstmt::Int, bb::Int) # inconsistent region. if !sv.result.ipo_effects.terminates sv.all_retpaths_consistent = false - elseif conditional_successors_may_throw(sv.lazypostdomtree, sv.ir, bb) - # Check if there are potential throws that require + elseif visit_conditional_successors(sv.lazypostdomtree, sv.ir, bb) do succ::Int + return any_stmt_may_throw(sv.ir, succ) + end + # check if this `GotoIfNot` leads to conditional throws, which taints consistency sv.all_retpaths_consistent = false else (; cfg, domtree) = get!(sv.lazyagdomtree) diff --git a/test/compiler/effects.jl b/test/compiler/effects.jl index f5ecb8a0d347c..fa70c8de9d853 100644 --- a/test/compiler/effects.jl +++ b/test/compiler/effects.jl @@ -1382,3 +1382,8 @@ let; Base.Experimental.@force_compile; func52843(); end # pointerref nothrow for invalid pointer @test !Core.Compiler.intrinsic_nothrow(Core.Intrinsics.pointerref, Any[Type{Ptr{Vector{Int64}}}, Int, Int]) @test !Core.Compiler.intrinsic_nothrow(Core.Intrinsics.pointerref, Any[Type{Ptr{T}} where T, Int, Int]) + +# post-opt :consistent-cy analysis correctness +# https://github.com/JuliaLang/julia/issues/53508 +@test !Core.Compiler.is_consistent(Base.infer_effects(getindex, (UnitRange{Int},Int))) +@test !Core.Compiler.is_consistent(Base.infer_effects(getindex, (Base.OneTo{Int},Int))) diff --git a/test/compiler/ssair.jl b/test/compiler/ssair.jl index 3a90ee6308d53..0a53dec80f732 100644 --- a/test/compiler/ssair.jl +++ b/test/compiler/ssair.jl @@ -238,6 +238,35 @@ let ci = code_lowered(()->@isdefined(_not_def_37919_), ())[1] @test Core.Compiler.verify_ir(ir) === nothing end +let code = Any[ + # block 1 + GotoIfNot(Argument(2), 4), + # block 2 + GotoNode(3), + # block 3 + Expr(:call, throw, "potential throw"), + # block 4 + Expr(:call, Core.Intrinsics.add_int, Argument(3), Argument(4)), + GotoNode(6), + # block 5 + ReturnNode(SSAValue(4)) + ] + ir = make_ircode(code; slottypes=Any[Any,Bool,Int,Int]) + lazypostdomtree = Core.Compiler.LazyPostDomtree(ir) + visited = BitSet() + @test !Core.Compiler.visit_conditional_successors(lazypostdomtree, ir, #=bb=#1) do succ::Int + push!(visited, succ) + return false + end + @test 2 ∈ visited + @test 3 ∈ visited + @test 4 ∉ visited + @test 5 ∉ visited + oc = Core.OpaqueClosure(ir) + @test oc(false, 1, 1) == 2 + @test_throws "potential throw" oc(true, 1, 1) +end + # Test dynamic update of domtree with edge insertions and deletions in the # following CFG: #