From 599a9b2c04837e9d2162a022baf3394376af0cd9 Mon Sep 17 00:00:00 2001 From: George Datseris Date: Fri, 22 Oct 2021 18:38:31 +0200 Subject: [PATCH] use environment variables for tagging (#290) * use environment variables for tagging Hope I did everything right! * fix tests * changelog --- CHANGELOG.md | 2 ++ Project.toml | 2 +- src/naming.jl | 8 +++----- src/saving_files.jl | 37 +++++++++++++++++++++++++------------ src/saving_tools.jl | 24 ++++++++++++++++++------ 5 files changed, 49 insertions(+), 24 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 34243965..299d21af 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,5 @@ +# 2.7.0 +* DrWatson-related `ENV`ironment variables are now available to globally set the default values for e.g. story git patches, tagging, or safe-saving in various functions like `tagsave` or `produce_or_load`. # 2.6.0 * Use `JLD2`'s jldopen in `collect_results!` to speed up loading of metadata. # 2.5.0 diff --git a/Project.toml b/Project.toml index d9925799..b5a7daad 100644 --- a/Project.toml +++ b/Project.toml @@ -1,7 +1,7 @@ name = "DrWatson" uuid = "634d3b9d-ee7a-5ddf-bec9-22491ea816e1" repo = "https://github.com/JuliaDynamics/DrWatson.jl.git" -version = "2.6.2" +version = "2.7.0" [deps] Dates = "ade2ca70-3891-5945-98fb-dc099432e06a" diff --git a/src/naming.jl b/src/naming.jl index 3a41e600..bb019cd6 100644 --- a/src/naming.jl +++ b/src/naming.jl @@ -91,11 +91,9 @@ function savename(prefix::String, c, suffix::String; sort = true, equals = "=") if any(sep in prefix for sep in ['/', '\\']) - @warn """ - Path separators in `savename` prefixes may break reproducibility on other OS. - The recommended way is using the `*dir()` methods or `joinpath` with - `savename` (e.g. `datadir("path", "to", "folder", savename("prefix", data))`). - """ + @warn "Path separators in `savename` prefixes may break reproducibility on other OS. "* + "The recommended way is using the `*dir()` methods or `joinpath` with "* + "`savename` (e.g. `datadir(\"path\", \"to\", \"folder\", savename(\"prefix\", data))`)." end sigdigits = digits === nothing ? sigdigits : nothing val2string = val_to_string === nothing ? (val -> valtostring(val, digits, sigdigits)) : val_to_string diff --git a/src/saving_files.jl b/src/saving_files.jl index a06dea7e..8eb1ee6b 100644 --- a/src/saving_files.jl +++ b/src/saving_files.jl @@ -24,7 +24,8 @@ end ## Keywords * `suffix = "jld2", prefix = default_prefix(config)` : Used in [`savename`](@ref). -* `tag::Bool = istaggable(suffix)` : Save the file using [`tagsave`](@ref) if `true`. +* `tag::Bool = get(ENV, "DRWATSON_TAG", istaggable(suffix))` : Save the file + using [`tagsave`](@ref) if `true` (which is the default). * `gitpath, storepatch` : Given to [`tagsave`](@ref) if `tag` is `true`. * `force = false` : If `true` then don't check if file `s` exists and produce it and save it anyway. @@ -41,9 +42,13 @@ produce_or_load(c, f; kwargs...) = produce_or_load("", c, f; kwargs...) produce_or_load(f::Function, c; kwargs...) = produce_or_load(c, f; kwargs...) produce_or_load(f::Function, path, c; kwargs...) = produce_or_load(path, c, f; kwargs...) function produce_or_load(path, c, f::Function; - suffix = "jld2", prefix = default_prefix(c), - tag::Bool = istaggable(suffix), gitpath = projectdir(), loadfile = true, - force = false, verbose = true, storepatch = true, wsave_kwargs = Dict(), kwargs...) + suffix = "jld2", prefix = default_prefix(c), + tag::Bool = get(ENV, "DRWATSON_TAG", istaggable(suffix)), + gitpath = projectdir(), loadfile = true, + storepatch::Bool = get(ENV, "DRWATSON_STOREPATCH", true), + force = false, verbose = true, wsave_kwargs = Dict(), + kwargs... + ) s = joinpath(path, savename(prefix, c, suffix; kwargs...)) @@ -118,21 +123,29 @@ end # tag saving # ################################################################################ """ - tagsave(file::String, d::AbstractDict; safe = false, gitpath = projectdir(), storepatch = true, force = false, kwargs...) + tagsave(file::String, d::AbstractDict; kwargs...) First [`tag!`](@ref) dictionary `d` and then save `d` in `file`. -If `safe = true` save the file using [`safesave`](@ref). "Tagging" means that when saving the dictionary, an extra field `:gitcommit` is added to establish reproducibility of results using -Git. If the Git repository is dirty, one more field `:gitpatch` is -added that stores the difference string. If a dictionary already -contains a key `:gitcommit`, it is not overwritten, unless, +Git. If the Git repository is dirty and `storepatch=true`, one more field `:gitpatch` is +added that stores the difference string. If a dictionary already +contains a key `:gitcommit`, it is not overwritten, unless `force=true`. For more details, see [`tag!`](@ref). +Keywords `gitpath, storepatch, force,` are propagated to [`tag!`](@ref). Any additional keyword arguments are propagated to `wsave`, to e.g. enable compression. + +The keyword `safe = get(ENV, "DRWATSON_SAFESAVE", false)` decides whether +to save the file using [`safesave`](@ref). """ -function tagsave(file, d; safe::Bool = false, gitpath = projectdir(), storepatch = true, force = false, source = nothing, kwargs...) +function tagsave(file, d; + gitpath = projectdir(), + safe::Bool = get(ENV, "DRWATSON_SAFESAVE", false), + storepatch::Bool = get(ENV, "DRWATSON_STOREPATCH", true), + force = false, source = nothing, kwargs... + ) d2 = tag!(d, gitpath=gitpath, storepatch=storepatch, force=force, source=source) if safe safesave(file, copy(d2); kwargs...) @@ -200,7 +213,7 @@ function increment_backup_num(filepath) path, filename = splitdir(filepath) fname, suffix = splitext(filename) m = match(r"^(.*)_#([0-9]+)$", fname) - if m == nothing + if m === nothing return joinpath(path, "$(fname)_#1$(suffix)") end newnum = string(parse(Int, m.captures[2]) +1) @@ -228,7 +241,7 @@ Save each entry in `dicts` into a unique temporary file in the directory `tmp`. Then return the list of file names (relative to `tmp`) that were used for saving each dictionary. Each dictionary can then be loaded back by calling - FileIO.load(nth_tmpfilename, "params") + wload(nth_tmpfilename, "params") `tmp` defaults to `projectdir("_research", "tmp")`. diff --git a/src/saving_tools.jl b/src/saving_tools.jl index 277450e5..65c9c546 100644 --- a/src/saving_tools.jl +++ b/src/saving_tools.jl @@ -141,7 +141,7 @@ function gitpatch(path = projectdir(); try_submodule_diff=true) # Remove the submodule option as it is not supported by older git versions. return gitpatch(path; try_submodule_diff = false) else - @warn "`gitpatch` failed with error $(result.err) $(result.exception) and , returning `nothing` instead." + @warn "`gitpatch` failed with error $(result.err) $(result.exception). Returning `nothing` instead." end catch er if isa(er,LibGit2.GitError) && er.code == LibGit2.Error.ENOTFOUND @@ -160,7 +160,7 @@ end # Tagging ######################################################################################## """ - tag!(d::AbstractDict; gitpath = projectdir(), storepatch = true, force = false) -> d + tag!(d::AbstractDict; kwargs...) -> d Tag `d` by adding an extra field `gitcommit` which will have as value the [`gitdescribe`](@ref) of the repository at `gitpath` (by default the project's gitpath). Do nothing if a key `gitcommit` already exists @@ -178,6 +178,12 @@ To restore a repository to the state of a particular model-run do: 1. checkout the relevant commit with `git checkout xyz` where xyz is the value stored 2. apply the patch `git apply patch`, where the string stored in the `gitpatch` field needs to be written to the file `patch`. +## Keywords +* `gitpath = projectdir()` +* `force = false` +* `storepatch = get(ENV, "DRWATSON_STOREPATCH", true)`: Whether to collect and store the + output of [`gitpatch`](@ref) as well. + ## Examples ```julia julia> d = Dict(:x => 3, :y => 4) @@ -192,7 +198,10 @@ Dict{Symbol,Any} with 3 entries: :x => 3 ``` """ -function tag!(d::AbstractDict{K,T}; gitpath = projectdir(), storepatch = true, force = false, source = nothing) where {K,T} +function tag!(d::AbstractDict{K,T}; + gitpath = projectdir(), force = false, source = nothing, + storepatch::Bool = get(ENV, "DRWATSON_STOREPATCH", true), + ) where {K,T} @assert (K <: Union{Symbol,String}) "We only know how to tag dictionaries that have keys that are strings or symbols" c = gitdescribe(gitpath) c === nothing && return d # gitpath is not a git repo @@ -210,7 +219,7 @@ function tag!(d::AbstractDict{K,T}; gitpath = projectdir(), storepatch = true, f # Only include patch info if `storepatch` is true and if we can get the info. if storepatch patch = gitpatch(gitpath) - if (patch != nothing) && (patch != "") + if (patch !== nothing) && (patch != "") d[patchname] = patch end end @@ -330,10 +339,13 @@ additional data fields as strings. Currently endings that can do this are: ``` $(TAGGABLE_FILE_ENDINGS) ``` - - istaggable(x) = x isa AbstractDictionary """ istaggable(file::AbstractString) = any(endswith(file, e) for e ∈ TAGGABLE_FILE_ENDINGS) + +""" + istaggable(x) = x isa AbstractDict +For non-string input the function just checks if input is dictionary. +""" istaggable(x) = x isa AbstractDict