Skip to content

Refactoring shortnames option #318

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 12 commits into from
Jan 23, 2022
Merged

Refactoring shortnames option #318

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
12 commits
Select commit Hold shift + click to select a range
7de1c68
Starting shortnames refactor
briochh Dec 19, 2021
bb8762a
More carefully handling shortname mapping for repeat pst builds in ps…
briochh Dec 21, 2021
2b33a32
need to pad tpl entries to prevent excess precision loss
briochh Dec 21, 2021
6b720bc
control file may not have parameter group section
briochh Dec 21, 2021
2123ba9
Tidy longname refs from pst_utils
briochh Dec 21, 2021
042a3b5
Tidy old long name refs from pst_from
briochh Dec 21, 2021
efe09ec
Tidy longname refs from pp_utils
briochh Dec 21, 2021
4352b11
Short name test can now be shorter
briochh Dec 21, 2021
b6c7061
Refactoring shortnames option in PstFrom (#312)
Dec 21, 2021
4c77063
Merge branch 'feat_shortnames' of https://github.com/pypest/pyemu int…
briochh Jan 20, 2022
8c15fd8
Merge branch 'develop' into feat_shortnames
briochh Jan 20, 2022
dcf3b20
Merge branch 'develop' into feat_shortnames
jtwhite79 Jan 23, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
162 changes: 44 additions & 118 deletions autotest/pst_from_tests.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -984,100 +984,6 @@ def mf6_freyberg_shortnames_test():
os.path.join(tmp_model_ws, "freyberg6.nam"),
delr=m.dis.delr.array, delc=m.dis.delc.array)
# set up PstFrom object
pf = PstFrom(original_d=tmp_model_ws, new_d=template_ws,
remove_existing=True,
longnames=False, spatial_reference=sr,
zero_based=False,start_datetime="1-1-2018")

df = pd.read_csv(os.path.join(tmp_model_ws, "heads.csv"), index_col=0)
try:
pf.add_observations("heads.csv", insfile="heads.csv.ins", index_cols="time", use_cols=list(df.columns.values),
prefix="hds_too_long_tho")
except:
pass
else:
raise Exception("should have failed")

pf = PstFrom(original_d=tmp_model_ws, new_d=template_ws,
remove_existing=True,
longnames=False, spatial_reference=sr,
zero_based=False, start_datetime="1-1-2018")
try:
pf.add_parameters(filenames="freyberg6.sfr_packagedata.txt", par_name_base="rhk_still_too_long",
pargp="sfr_rhk", index_cols=[0, 1, 2, 3], use_cols=[9], upper_bound=10., lower_bound=0.1,
par_type="grid")
except:
pass
else:
raise Exception("should have failed")

pf = PstFrom(original_d=tmp_model_ws, new_d=template_ws,
remove_existing=True,
longnames=False, spatial_reference=sr,
zero_based=False, start_datetime="1-1-2018")
try:
pf.add_parameters(filenames="freyberg6.sfr_packagedata.txt", par_name_base="rhk_still_too_long",
pargp="sfr_rhk", index_cols=[0, 1, 2, 3], use_cols=[9], upper_bound=10., lower_bound=0.1,
par_type="grid", par_style="direct")
except:
pass
else:
raise Exception("should have failed")

pf = PstFrom(original_d=tmp_model_ws, new_d=template_ws,
remove_existing=True,
longnames=False, spatial_reference=sr,
zero_based=False, start_datetime="1-1-2018")
try:
pf.add_parameters(filenames="freyberg6.ic_strt_layer1.txt", par_name_base="rhk_still_too_long",
pargp="sfr_rhk", upper_bound=10., lower_bound=0.1,
par_type="pilotpoints")
except:
pass
else:
raise Exception("should have failed")

pf = PstFrom(original_d=tmp_model_ws, new_d=template_ws,
remove_existing=True,
longnames=False, spatial_reference=sr,
zero_based=False, start_datetime="1-1-2018")
try:
pf.add_parameters(filenames="freyberg6.ic_strt_layer1.txt", par_name_base="rhk_still_too_long",
pargp="sfr_rhk", upper_bound=10., lower_bound=0.1,
par_type="grid")
except:
pass
else:
raise Exception("should have failed")

try:
pf.add_parameters(filenames="freyberg6.ic_strt_layer1.txt", par_name_base="rhk_still_too_long",
pargp="sfr_rhk", upper_bound=10., lower_bound=0.1,
par_type="grid",par_style="direct")
except:
pass
else:
raise Exception("should have failed")


try:
pf.add_parameters(filenames="freyberg6.ic_strt_layer1.txt", par_name_base="rhk_still_too_long",
pargp="sfr_rhk", upper_bound=10., lower_bound=0.1,
par_type="constant")
except:
pass
else:
raise Exception("should have failed")

try:
pf.add_parameters(filenames="freyberg6.ic_strt_layer1.txt", par_name_base="rhk_still_too_long",
pargp="sfr_rhk", upper_bound=10., lower_bound=0.1,
par_type="zone")
except:
pass
else:
raise Exception("should have failed")

# obs
# using tabular style model output
# (generated by pyemu.gw_utils.setup_hds_obs())
Expand All @@ -1088,11 +994,10 @@ def mf6_freyberg_shortnames_test():
remove_existing=True,
longnames=False, spatial_reference=sr,
zero_based=False, start_datetime="1-1-2018")
df = pd.read_csv(os.path.join(tmp_model_ws,"heads.csv"),index_col=0)
pf.add_observations("heads.csv",insfile="heads.csv.ins",index_cols="time",use_cols=list(df.columns.values),prefix="hds")
df = pd.read_csv(os.path.join(tmp_model_ws, "sfr.csv"), index_col=0)
pf.add_observations("sfr.csv", insfile="sfr.csv.ins", index_cols="time", use_cols=list(df.columns.values))
v = pyemu.geostats.ExpVario(contribution=1.0,a=1000)
df = pd.read_csv(os.path.join(tmp_model_ws,"heads.csv"), index_col=0)
pf.add_observations("heads.csv",insfile="heads.csv.ins", index_cols="time",
use_cols=list(df.columns.values), prefix="hds")
v = pyemu.geostats.ExpVario(contribution=1.0, a=1000)
gr_gs = pyemu.geostats.GeoStruct(variograms=v)
rch_temporal_gs = pyemu.geostats.GeoStruct(variograms=pyemu.geostats.ExpVario(contribution=1.0,a=60))
pf.extra_py_imports.append('flopy')
Expand Down Expand Up @@ -1129,7 +1034,7 @@ def mf6_freyberg_shortnames_test():
pf.add_parameters(filenames=list_file,par_type="constant",par_name_base="w{0}".format(kper),
pargp="wel_{0}".format(kper),index_cols=[0,1,2],use_cols=[3],
upper_bound=1.5,lower_bound=0.5)
za = np.ones((3,40,20))
za = np.ones((3, 40, 20))
df = pd.read_csv(os.path.join(m.model_ws, list_file),
delim_whitespace=True, header=None) - 1
za[tuple(df.loc[0:2, [0, 1, 2]].values.T)] = [2,3,4]
Expand All @@ -1140,9 +1045,6 @@ def mf6_freyberg_shortnames_test():
upper_bound=1.5, lower_bound=0.5,
zone_array=za)
assert len(pdf) == 4
pf.add_parameters(filenames="freyberg6.sfr_packagedata.txt", par_name_base="rhk",
pargp="sfr_rhk", index_cols=[0, 1, 2, 3], use_cols=[9], upper_bound=10., lower_bound=0.1,
par_type="grid")

# add model run command
pf.mod_sys_cmds.append("mf6")
Expand All @@ -1152,6 +1054,18 @@ def mf6_freyberg_shortnames_test():
# build pest
pst = pf.build_pst('freyberg.pst')

# test update/rebuild
pf.add_parameters(filenames="freyberg6.sfr_packagedata.txt",
par_name_base="rhk",
pargp="sfr_rhk", index_cols=[0, 1, 2, 3], use_cols=[9],
upper_bound=10., lower_bound=0.1,
par_type="grid", rebuild_pst=True)
pf.add_parameters(filenames=arr_file, par_type="grid", par_name_base=pb + "g2",
pargp=pb + "g2", zone_array=ib, upper_bound=ub, lower_bound=lb,
geostruct=gr_gs, rebuild_pst=True)
df = pd.read_csv(os.path.join(tmp_model_ws, "sfr.csv"), index_col=0)
pf.add_observations("sfr.csv", insfile="sfr.csv.ins", index_cols="time",
use_cols=list(df.columns.values), rebuild_pst=True)
assert pst.parameter_data.parnme.apply(lambda x: len(x)).max() <= 12
assert pst.observation_data.obsnme.apply(lambda x: len(x)).max() <= 20

Expand All @@ -1171,13 +1085,14 @@ def mf6_freyberg_shortnames_test():
os.chdir(b_d)
raise Exception(str(e))
os.chdir(b_d)

pst.try_parse_name_metadata()
pst.control_data.noptmax = 0
pst.pestpp_options["additional_ins_delimiters"] = ","

pst.write(os.path.join(pf.new_d, "freyberg.pst"))
pyemu.os_utils.run("{0} freyberg.pst".format(ies_exe_path), cwd=pf.new_d)

pst = pyemu.Pst(os.path.join(pf.new_d, "freyberg.pst"))
res_file = os.path.join(pf.new_d, "freyberg.base.rei")
assert os.path.exists(res_file), res_file
pst.set_res(res_file)
Expand Down Expand Up @@ -1592,7 +1507,8 @@ def mf6_freyberg_direct_test():
cov.to_coo("prior.jcb")
pst.try_parse_name_metadata()
df = pd.read_csv(os.path.join(tmp_model_ws, "heads.csv"), index_col=0)
pf.add_observations("heads.csv", insfile="heads.csv.ins", index_cols="time", use_cols=list(df.columns.values),
pf.add_observations("heads.csv", insfile="heads.csv.ins", index_cols="time",
use_cols=list(df.columns.values),
prefix="hds", rebuild_pst=True)

# test par mults are working
Expand Down Expand Up @@ -1992,8 +1908,13 @@ def mf6_freyberg_short_direct_test():
rch_temporal_gs = pyemu.geostats.GeoStruct(variograms=pyemu.geostats.ExpVario(contribution=1.0, a=60))
pf.extra_py_imports.append('flopy')
ib = m.dis.idomain[0].array
tags = {"npf_k_": [0.1, 10.], "npf_k33_": [.1, 10], "sto_ss": [.1, 10], "sto_sy": [.9, 1.1],
"rch_recharge": [.5, 1.5]}
tags = {
"npf_k_": [0.1, 10.],
"npf_k33_": [.1, 10],
"sto_ss": [.1, 10],
"sto_sy": [.9, 1.1],
"rch_recharge": [.5, 1.5]
}
dts = pd.to_datetime("1-1-2018") + pd.to_timedelta(np.cumsum(sim.tdis.perioddata.array["perlen"]), unit="d")
print(dts)
# setup from array style pars
Expand Down Expand Up @@ -2064,7 +1985,7 @@ def mf6_freyberg_short_direct_test():
pargp="wel_{0}_direct".format(kper), index_cols=[0, 1, 2], use_cols=[3],
upper_bound=0.0, lower_bound=-1000, geostruct=gr_gs, par_style="direct",
transform="none")
# Adding dummy list pars with different file structures
# # Adding dummy list pars with different file structures
list_file = "new_freyberg6.wel_stress_period_data_1.txt" # with a header
pf.add_parameters(filenames=list_file, par_type="grid", par_name_base='nw',
pargp='nwell_mult', index_cols=['k', 'i', 'j'], use_cols='flx',
Expand Down Expand Up @@ -2116,7 +2037,8 @@ def mf6_freyberg_short_direct_test():
cov.to_coo("prior.jcb")
pst.try_parse_name_metadata()
df = pd.read_csv(os.path.join(tmp_model_ws, "heads.csv"), index_col=0)
pf.add_observations("heads.csv", insfile="heads.csv.ins", index_cols="time", use_cols=list(df.columns.values),
pf.add_observations("heads.csv", insfile="heads.csv.ins", index_cols="time",
use_cols=list(df.columns.values),
prefix="hds", rebuild_pst=True)

# test par mults are working
Expand Down Expand Up @@ -2166,13 +2088,16 @@ def mf6_freyberg_short_direct_test():
# turn direct recharge to min and direct wel to min and
# check that the model results are consistent
par = pst.parameter_data
rch_par = par.loc[par.pargp.apply(
lambda x: "rch_gr" in x and "direct" in x), "parnme"]
wel_par = par.loc[par.pargp.apply(
lambda x: "wel_" in x and "direct" in x), "parnme"]
par.loc[rch_par,"parval1"] = par.loc[rch_par, "parlbnd"]
rch_par = par.loc[(par.pname == 'rch') &
(par.ptype == 'gr') &
(par.pstyle == 'd'),
"parnme"]
wel_par = par.loc[(par.pname.str.contains('wel')) &
(par.pstyle == 'd'),
"parnme"]
par.loc[rch_par, "parval1"] = par.loc[rch_par, "parlbnd"]
# this should set wells to zero since they are negative values in the control file
par.loc[wel_par,"parval1"] = par.loc[wel_par, "parubnd"]
par.loc[wel_par, "parval1"] = par.loc[wel_par, "parubnd"]
pst.write(os.path.join(pf.new_d, "freyberg.pst"))
pyemu.os_utils.run("{0} freyberg.pst".format(ies_exe_path), cwd=pf.new_d)
lst = flopy.utils.Mf6ListBudget(os.path.join(pf.new_d, "freyberg6.lst"))
Expand Down Expand Up @@ -2853,7 +2778,7 @@ def mf6_freyberg_arr_obs_and_headerless_test():
os.mkdir(tmp_model_ws)
sim = flopy.mf6.MFSimulation.load(sim_ws=org_model_ws)
# sim.set_all_data_external()
sim.simulation_data.mfpath.set_sim_path(tmp_model_ws)
sim.set_sim_path(tmp_model_ws)
# sim.set_all_data_external()
m = sim.get_model("freyberg6")
sim.set_all_data_external(check_data=False)
Expand Down Expand Up @@ -3688,19 +3613,20 @@ def mf6_subdir_test():
# assert np.abs(float(df.upper_bound.min()) - 30.) < 1.0e-6,df.upper_bound.min()
# assert np.abs(float(df.lower_bound.max()) - -0.3) < 1.0e-6,df.lower_bound.max()


if __name__ == "__main__":
#mf6_freyberg_pp_locs_test()
# invest()
freyberg_test()
#freyberg_prior_build_test()
# mf6_freyberg_test()
#$mf6_freyberg_da_test()
#mf6_freyberg_shortnames_test()
# mf6_freyberg_shortnames_test()

#mf6_freyberg_direct_test()
#mf6_freyberg_varying_idomain()
# xsec_test()
# mf6_freyberg_short_direct_test()
mf6_freyberg_short_direct_test()
# mf6_add_various_obs_test()
# mf6_subdir_test()
tpf = TestPstFrom()
Expand Down
38 changes: 35 additions & 3 deletions pyemu/pst/pst_handler.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1243,8 +1243,28 @@ def load(self, filename):
)

self._load_version2(filename)
self._try_load_longnames()
self.try_parse_name_metadata()

def _try_load_longnames(self):
from pathlib import Path
d = Path(self.filename).parent
for df, fnme in ((self.parameter_data, "parlongname.map"),
(self.observation_data, "obslongname.map")):
try:
mapr = pd.read_csv(Path(d, fnme), index_col=0)['longname']
df['longname'] = df.index.map(mapr.to_dict())
except Exception:
pass
if hasattr(self, "parameter_groups"):
df, fnme = (self.parameter_groups, "pglongname.map")
try:
mapr = pd.read_csv(Path(d, fnme), index_col=0)['longname']
df['longname'] = df.index.map(mapr.to_dict())
except Exception:
pass


def _parse_pestpp_line(self, line):
# args = line.replace('++','').strip().split()
args = line.replace("++", "").strip().split(")")
Expand Down Expand Up @@ -3751,8 +3771,17 @@ def try_parse_name_metadata(self):
par_cols = pst_utils.pst_config["par_fieldnames"]
obs_cols = pst_utils.pst_config["obs_fieldnames"]

if "longname" in par.columns:
partg = "longname"
else:
partg = "parnme"
if "longname" in obs.columns:
obstg = "longname"
else:
obstg = "obsnme"

for df, name, fieldnames in zip(
[par, obs], ["parnme", "obsnme"], [par_cols, obs_cols]
[par, obs], [partg, obstg], [par_cols, obs_cols]
):
try:
meta_dict = df.loc[:, name].apply(
Expand Down Expand Up @@ -3817,9 +3846,12 @@ def rename_parameters(self, name_dict, pst_path="."):
eq = eq.replace(old, new)
self.prior_information.loc[idx, "equation"] = eq

# pad for putting to tpl
name_dict = {k: v.center(12) for k, v in name_dict.items()}
for tpl_file in self.model_input_data.pest_file:
sys_tpl_file = os.path.join(
pst_path, tpl_file.replace("/", os.path.sep).replace("\\", os.path.sep)
pst_path,
str(tpl_file).replace("/", os.path.sep).replace("\\", os.path.sep)
)
if not os.path.exists(sys_tpl_file):
warnings.warn(
Expand Down Expand Up @@ -3869,7 +3901,7 @@ def rename_observations(self, name_dict, pst_path="."):

for ins_file in self.model_output_data.pest_file:
sys_ins_file = os.path.join(
pst_path, ins_file.replace("/", os.path.sep).replace("\\", os.path.sep)
pst_path, str(ins_file).replace("/", os.path.sep).replace("\\", os.path.sep)
)
if not os.path.exists(sys_ins_file):
warnings.warn(
Expand Down
Loading