From 11a6195bb5e66cfdee83632a5779237d9e46cd8c Mon Sep 17 00:00:00 2001 From: Louis Ponet Date: Fri, 2 Jul 2021 12:02:57 +0200 Subject: [PATCH] improved rm_version and added scf_steps to filereading --- src/job.jl | 13 ++++++++++-- src/jobAPI.jl | 2 +- src/qe/fileio.jl | 17 ++++++---------- src/types.jl | 13 +++++++----- src/versioning.jl | 52 ++++++++++++++++++++++++++++++++++++----------- 5 files changed, 66 insertions(+), 31 deletions(-) diff --git a/src/job.jl b/src/job.jl index 1b44b167..fd1e7e12 100644 --- a/src/job.jl +++ b/src/job.jl @@ -242,6 +242,7 @@ function isrunning(job::DFJob; print = true) return false end pwd = split(strip(read(`pwdx $(pids[end])`, String)))[end] + @show pwd return abspath(pwd) == job.local_dir catch return false @@ -276,7 +277,7 @@ function cleanup(job::DFJob) labels = String[] paths = String[] for v in versions(job) - vpath = version_path(job, v) + vpath = version_dir(job, v) s = round(dirsize(vpath) / 1e6; digits = 3) push!(labels, "Version $v: $s Mb") push!(paths, vpath) @@ -315,4 +316,12 @@ function clean_local_dir!(dir::AbstractString) end end -main_job_dir(job::DFJob) = split(job.local_dir, VERSION_DIR_NAME)[1] +""" + main_job_dir(dir::AbstractString) + main_job_dir(job::DFJob) + +Returns the main directory of the job, also when the job's version is not the one +in the main directory. +""" +main_job_dir(dir::AbstractString) = split(dir, VERSION_DIR_NAME)[1] +main_job_dir(job::DFJob) = main_job_dir(job.local_dir) diff --git a/src/jobAPI.jl b/src/jobAPI.jl index 7b24b808..46ce1d6f 100644 --- a/src/jobAPI.jl +++ b/src/jobAPI.jl @@ -16,7 +16,7 @@ function save(job::DFJob; kwargs...) tj = DFJob(local_dir) @assert !isrunning(tj) "Can't save a job in a directory where another is running." # We want to first make sure that the previous job in the main directory is safely stored - cp(tj, version_path(tj); force = true) + cp(tj, joinpath(tj, VERSION_DIR_NAME, "$(tj.version)"); force = true) end if local_dir != job.local_dir # We know for sure it was a previously saved job diff --git a/src/qe/fileio.jl b/src/qe/fileio.jl index 5f94104b..7b38d2e1 100644 --- a/src/qe/fileio.jl +++ b/src/qe/fileio.jl @@ -289,6 +289,10 @@ function qe_parse_scf_iteration(out, line, f) else push!(out[:scf_iteration], it) end + if it == 1 + out[:scf_converged] = false + haskey(out, :scf_steps) ? out[:scf_steps] += 1 : out[:scf_steps] = 1 + end end function qe_parse_colin_magmoms(out, line, f) @@ -321,14 +325,6 @@ function qe_parse_total_magnetization(out, line, f) end end -function qe_parse_scf_convergence(out, line, f) - if occursin("NOT", line) - out[:scf_converged] = false - elseif occursin("has been", line) - out[:scf_converged] = true - end -end - function qe_parse_magnetization(out, line, f) if !haskey(out, :magnetization) out[:magnetization] = Vec3{Float64}[] @@ -419,7 +415,7 @@ const QE_PW_PARSE_FUNCTIONS = ["C/m^2" => qe_parse_polarization, "Magnetic moment per site" => qe_parse_colin_magmoms, "estimated scf accuracy" => qe_parse_scf_accuracy, "total magnetization" => qe_parse_total_magnetization, - "convergence" => qe_parse_scf_convergence, + "convergence has been" => (x, y, z) -> x[:scf_converged] = true, "Begin final coordinates" => (x, y, z) -> x[:converged] = true, "atom number" => qe_parse_magnetization, "--- enter write_ns ---" => qe_parse_Hubbard, @@ -988,8 +984,7 @@ function qe_read_calculation(filename; execs = [Exec(; exec = "pw.x")], run = tr parsed_flags[sym][ids...] = length(parsedval) == 1 ? parsedval[1] : parsedval catch e - @warn "Parsing error of flag $f in file $filename." exception = (e, - catch_backtrace()) + @warn "Parsing error of flag $f in file $filename." exception = e end end structure = extract_structure!(structure_name, parsed_flags, cell_block, atsyms, diff --git a/src/types.jl b/src/types.jl index a1003a35..ae7b42bc 100644 --- a/src/types.jl +++ b/src/types.jl @@ -382,21 +382,24 @@ end function DFJob(job_dir::AbstractString, job_script = "job.tt"; version::Union{Nothing,Int} = nothing, kwargs...) apath = abspath(job_dir) - if !isempty(job_dir) && ispath(apath) && !isempty(searchdir(apath, job_script)) + if ispath(apath) if occursin(VERSION_DIR_NAME, apath) @error "It is not allowed to directly load a job version, please use `DFJob(dir, version=$(splitdir(apath)[end]))`" end - if version === nothing || (ispath(joinpath(apath, ".metadata.jld2")) && - load(joinpath(apath, ".metadata.jld2"))["version"] == version) + if version !== nothing + real_path = version_dir(apath, version) + real_version = version + elseif ispath(joinpath(apath, job_script)) real_path = apath + real_version = main_job_version(apath) else - real_path = joinpath(apath, VERSION_DIR_NAME, "$version") + @error "No valid job found in $apath." end else real_path = request_job(job_dir) real_path === nothing && return + real_version = main_job_version(real_path) end - real_version = version === nothing ? last_job_version(real_path) : version return DFJob(; merge(merge((local_dir = real_path, version = real_version), read_job_calculations(joinpath(real_path, job_script))), diff --git a/src/versioning.jl b/src/versioning.jl index 9537d90a..f5b3da09 100644 --- a/src/versioning.jl +++ b/src/versioning.jl @@ -2,7 +2,8 @@ const VERSION_DIR_NAME = ".versions" # 0 Job version means that no jobs have ran yet. "Returns the version found in the .metadata.jld2 if it exists. Otherwise 0." function main_job_version(dir::AbstractString) - mdatapath = joinpath(dir, ".metadata.jld2") + maindir = main_job_dir(dir) + mdatapath = joinpath(maindir, ".metadata.jld2") if ispath(mdatapath) metadata = load(mdatapath) if haskey(metadata, "version") @@ -11,6 +12,7 @@ function main_job_version(dir::AbstractString) end return 0 end +main_job_version(job::DFJob) = main_job_version(job.local_dir) function job_versions(dir::AbstractString) versions = Int[] @@ -43,26 +45,32 @@ Returns the last version number of `job`. """ last_version(job::DFJob) = last_job_version(main_job_dir(job)) -function version_path(dir::AbstractString, version::Int) - return joinpath(dir, VERSION_DIR_NAME, "$version") +function version_dir(dir::AbstractString, version::Int) + tpath = joinpath(dir, VERSION_DIR_NAME, "$version") + ispath(tpath) && return tpath + + if main_job_version(dir) == version + return dir + end + error("Version $version not found.") end -version_path(job::DFJob) = version_path(main_job_dir(job), job.version) -version_path(job::DFJob, version::Int) = version_path(main_job_dir(job), version) +version_dir(job::DFJob) = version_dir(main_job_dir(job), job.version) +version_dir(job::DFJob, version::Int) = version_dir(main_job_dir(job), version) exists_version(dir::AbstractString, version::Int) = version ∈ job_versions(dir) exists_version(job::DFJob, version::Int) = exists_version(main_job_dir(job), version) """ - maybe_cp_prev_version(job::DFJob) + maybe_cp_main_version(job::DFJob) -Looks in the `job.local_dir` for a previous version of the job, and copies it to the +Looks in the `job.local_dir` for the version of the job in the main directory, and copies it to the respective directory in the `.versions`. """ -function maybe_cp_prev_version(job::DFJob) +function maybe_cp_main_version(job::DFJob) maindir = main_job_dir(job) if ispath(joinpath(maindir, "job.tt")) tjob = DFJob(maindir) - cp(tjob, version_path(tjob); force = true) + cp(tjob, joinpath(tjob, VERSION_DIR_NAME, "$(job.version)"); force = true) end end @@ -88,7 +96,7 @@ function switch_version!(job::DFJob) vs = versions(job) timestamps = [] for v in vs - mdatapath = joinpath(version_path(job, v), ".metadata.jld2") + mdatapath = joinpath(version_dir(job, v), ".metadata.jld2") if ispath(mdatapath) && haskey(load(mdatapath), "metadata") && haskey(load(mdatapath)["metadata"], :timestamp) @@ -119,6 +127,27 @@ Removes the specified `versions` from the `job` if they exist. """ function rm_version!(job::DFJob, version::Int) version_assert(job, version) + if version == main_job_version(job) + for f in readdir(job.local_dir) + if occursin(".workflow", f) || f == VERSION_DIR_NAME || splitext(f)[2] == ".jl" + continue + else + rm(joinpath(job, f)) + end + end + md = main_job_dir(job) + lv = last_version(job) + if lv == version + lv = versions(job)[end-1] + end + if lv != 0 + tj = DFJob(md, version=lv) + cp(tj, md, force=true) + end + + else + rm(version_dir(job, version); recursive = true) + end if version == job.version @warn "Job version is the same as the one to be removed, switching to last known version." lv = last_version(job) @@ -127,7 +156,6 @@ function rm_version!(job::DFJob, version::Int) end switch_version!(job, lv) end - return rm(version_path(job, version); recursive = true) end function rm_versions!(job::DFJob, versions::Int...) @@ -140,6 +168,6 @@ end function rm_tmp_dirs!(job, vers = versions(job)...) for v in vers version_assert(job, v) - rm(joinpath(version_path(job, v), TEMP_CALC_DIR); recursive = true) + rm(joinpath(version_dir(job, v), TEMP_CALC_DIR); recursive = true) end end