From 6999166f356c804b1f0f5e45200cae5c2030c9aa Mon Sep 17 00:00:00 2001 From: Jeremy White Date: Fri, 19 Feb 2021 16:46:22 -0700 Subject: [PATCH 1/7] added string zone array test --- autotest/pst_from_tests.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/autotest/pst_from_tests.py b/autotest/pst_from_tests.py index a5ef977e1..361cf675d 100644 --- a/autotest/pst_from_tests.py +++ b/autotest/pst_from_tests.py @@ -172,7 +172,9 @@ def freyberg_test(): assert all([gp in obsnmes for gp in ['qaquifer', 'qout']]) pf.post_py_cmds.append( "sfodf.sort_index(1).to_csv('freyberg.sfo.csv', sep=',', index_label='idx')") - + zone_array = np.arange(m.nlay*m.nrow*m.ncol) + s = lambda x: "zval_"+str(x) + zone_array = np.array([s(x) for x in zone_array]).reshape(m.nlay,m.nrow,m.ncol) # pars pf.add_parameters(filenames="RIV_0000.dat", par_type="grid", index_cols=[0, 1, 2], use_cols=[3, 5], @@ -184,7 +186,7 @@ def freyberg_test(): pf.add_parameters(filenames=["WEL_0000.dat", "WEL_0001.dat"], par_type="grid", index_cols=[0, 1, 2], use_cols=3, par_name_base="welflux_grid", - zone_array=m.bas6.ibound.array) + zone_array=zone_array) pf.add_parameters(filenames=["WEL_0000.dat"], par_type="constant", index_cols=[0, 1, 2], use_cols=3, par_name_base=["flux_const"]) @@ -2670,10 +2672,10 @@ def mf6_freyberg_arr_obs_and_headerless_test(): #mf6_freyberg_varying_idomain() #xsec_test() #mf6_freyberg_short_direct_test() - tpf = TestPstFrom() - tpf.setup() - tpf.test_add_direct_array_parameters() - tpf.add + #tpf = TestPstFrom() + #tpf.setup() + #tpf.test_add_direct_array_parameters() + #tpf.add #pstfrom_profile() #mf6_freyberg_arr_obs_and_headerless_test()\ From 19ff27eb94160426f1456590ff1aa62bf4fa2cd0 Mon Sep 17 00:00:00 2001 From: Jeremy White Date: Fri, 19 Feb 2021 17:02:17 -0700 Subject: [PATCH 2/7] added string zone array test --- autotest/pst_from_tests.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/autotest/pst_from_tests.py b/autotest/pst_from_tests.py index 361cf675d..bedc6eafc 100644 --- a/autotest/pst_from_tests.py +++ b/autotest/pst_from_tests.py @@ -187,6 +187,10 @@ def freyberg_test(): par_type="grid", index_cols=[0, 1, 2], use_cols=3, par_name_base="welflux_grid", zone_array=zone_array) + pf.add_parameters(filenames="WEL_0000.dat", + par_type="grid", index_cols=[0, 1, 2], use_cols=3, + par_name_base="welflux_grid_direct", + zone_array=zone_array,par_style="direct") pf.add_parameters(filenames=["WEL_0000.dat"], par_type="constant", index_cols=[0, 1, 2], use_cols=3, par_name_base=["flux_const"]) @@ -215,6 +219,7 @@ def freyberg_test(): # check mult files are in pst input files csv = os.path.join(template_ws, "mult2model_info.csv") df = pd.read_csv(csv, index_col=0) + df = df.loc[pd.notna(df.mlt_file),:] pst_input_files = {str(f) for f in pst.input_files} mults_not_linked_to_pst = ((set(df.mlt_file.unique()) - pst_input_files) - From 43a47b0ae332c01a7dfe5842ef04166aa7958df9 Mon Sep 17 00:00:00 2001 From: unknown Date: Fri, 26 Feb 2021 08:57:34 -0700 Subject: [PATCH 3/7] added nan traps in Pst new format writer --- pyemu/pst/pst_handler.py | 99 +++++++++++++++++++++++++++++++++------- pyemu/utils/pst_from.py | 3 +- 2 files changed, 85 insertions(+), 17 deletions(-) diff --git a/pyemu/pst/pst_handler.py b/pyemu/pst/pst_handler.py index 275fd4600..1584242b9 100644 --- a/pyemu/pst/pst_handler.py +++ b/pyemu/pst/pst_handler.py @@ -1520,24 +1520,64 @@ def _write_version2(self, new_filename, use_pst_path=True, pst_rel_path="."): v = ",".join([str(vv) for vv in list(v)]) f_out.write("{0:30} {1}\n".format(k, v)) + + + # parameter groups + name = "pargp_data" + columns = self.pargp_fieldnames + if self.parameter_groups.loc[:, columns].isnull().values.any(): + # warnings.warn("WARNING: NaNs in {0} dataframe".format(name)) + csv_name = "pst.{0}.nans.csv".format( + name.replace(" ", "_").replace("*", "") + ) + df.to_csv(csv_name) + raise Exception( + "NaNs in {0} dataframe, csv written to {1}".format(name, csv_name) + ) f_out.write("* parameter groups external\n") - pargp_filename = new_filename.lower().replace(".pst", ".pargrp_data.csv") + pargp_filename = new_filename.lower().replace(".pst", ".{0}.csv".format(name)) if pst_path is not None: pargp_filename = os.path.join(pst_path, os.path.split(pargp_filename)[-1]) self.parameter_groups.to_csv(pargp_filename, index=False) pargp_filename = os.path.join(pst_rel_path, os.path.split(pargp_filename)[-1]) f_out.write("{0}\n".format(pargp_filename)) + + # parameter data + name = "par_data" + columns = self.par_fieldnames + if self.parameter_data.loc[:, columns].isnull().values.any(): + # warnings.warn("WARNING: NaNs in {0} dataframe".format(name)) + csv_name = "pst.{0}.nans.csv".format( + name.replace(" ", "_").replace("*", "") + ) + df.to_csv(csv_name) + raise Exception( + "NaNs in {0} dataframe, csv written to {1}".format(name, csv_name) + ) f_out.write("* parameter data external\n") - par_filename = new_filename.lower().replace(".pst", ".par_data.csv") + par_filename = new_filename.lower().replace(".pst", ".{0}.csv".format(name)) if pst_path is not None: par_filename = os.path.join(pst_path, os.path.split(par_filename)[-1]) self.parameter_data.to_csv(par_filename, index=False) par_filename = os.path.join(pst_rel_path, os.path.split(par_filename)[-1]) f_out.write("{0}\n".format(par_filename)) + + # observation data + name = "obs_data" + columns = self.obs_fieldnames + if self.observation_data.loc[:, columns].isnull().values.any(): + # warnings.warn("WARNING: NaNs in {0} dataframe".format(name)) + csv_name = "pst.{0}.nans.csv".format( + name.replace(" ", "_").replace("*", "") + ) + df.to_csv(csv_name) + raise Exception( + "NaNs in {0} dataframe, csv written to {1}".format(name, csv_name) + ) f_out.write("* observation data external\n") - obs_filename = new_filename.lower().replace(".pst", ".obs_data.csv") + obs_filename = new_filename.lower().replace(".pst", ".{0}.csv".format(name)) if pst_path is not None: obs_filename = os.path.join(pst_path, os.path.split(obs_filename)[-1]) self.observation_data.to_csv(obs_filename, index=False) @@ -1548,35 +1588,62 @@ def _write_version2(self, new_filename, use_pst_path=True, pst_rel_path="."): for mc in self.model_command: f_out.write("{0}\n".format(mc)) + # model input + name = "tplfile_data" + columns = self.model_io_fieldnames + if self.model_input_data.loc[:, columns].isnull().values.any(): + # warnings.warn("WARNING: NaNs in {0} dataframe".format(name)) + csv_name = "pst.{0}.nans.csv".format( + name.replace(" ", "_").replace("*", "") + ) + df.to_csv(csv_name) + raise Exception( + "NaNs in {0} dataframe, csv written to {1}".format(name, csv_name) + ) f_out.write("* model input external\n") - io_filename = new_filename.lower().replace(".pst", ".tplfile_data.csv") + io_filename = new_filename.lower().replace(".pst", ".{0}.csv".format(tplfile_data)) if pst_path is not None: io_filename = os.path.join(pst_path, os.path.split(io_filename)[-1]) - # pfiles = self.template_files - # pfiles.extend(self.instruction_files) - # mfiles = self.input_files - # mfiles.extend(self.output_files) - # io_df = pd.DataFrame({"pest_file": pfiles, "model_file": mfiles}) - # io_df.to_csv(io_filename, index=False) self.model_input_data.to_csv(io_filename, index=False) io_filename = os.path.join(pst_rel_path, os.path.split(io_filename)[-1]) f_out.write("{0}\n".format(io_filename)) + # model output + name = "insfile_data" + columns = self.model_io_fieldnames + if self.model_output_data.loc[:, columns].isnull().values.any(): + # warnings.warn("WARNING: NaNs in {0} dataframe".format(name)) + csv_name = "pst.{0}.nans.csv".format( + name.replace(" ", "_").replace("*", "") + ) + df.to_csv(csv_name) + raise Exception( + "NaNs in {0} dataframe, csv written to {1}".format(name, csv_name) + ) f_out.write("* model output external\n") - io_filename = new_filename.lower().replace(".pst", ".insfile_data.csv") + io_filename = new_filename.lower().replace(".pst", ".{0}.csv".format(name)) if pst_path is not None: io_filename = os.path.join(pst_path, os.path.split(io_filename)[-1]) - # pfiles = self.instruction_files - # mfiles = self.output_files - # io_df = pd.DataFrame({"pest_file": pfiles, "model_file": mfiles}) - # io_df.to_csv(io_filename, index=False) self.model_output_data.to_csv(io_filename, index=False) io_filename = os.path.join(pst_rel_path, os.path.split(io_filename)[-1]) f_out.write("{0}\n".format(io_filename)) + + # prior info if self.prior_information.shape[0] > 0: + name = "pi_data" + columns = self.prior_fieldnames + if self.prior_information.loc[:, columns].isnull().values.any(): + # warnings.warn("WARNING: NaNs in {0} dataframe".format(name)) + csv_name = "pst.{0}.nans.csv".format( + name.replace(" ", "_").replace("*", "") + ) + df.to_csv(csv_name) + raise Exception( + "NaNs in {0} dataframe, csv written to {1}".format(name, csv_name) + ) f_out.write("* prior information external\n") - pi_filename = new_filename.lower().replace(".pst", ".pi_data.csv") + pi_filename = new_filename.lower().replace(".pst", ".{0}.csv".format(name)) if pst_path is not None: pi_filename = os.path.join(pst_path, os.path.split(pi_filename)[-1]) self.prior_information.to_csv(pi_filename, index=False) diff --git a/pyemu/utils/pst_from.py b/pyemu/utils/pst_from.py index 6c3615f78..7ac55b868 100644 --- a/pyemu/utils/pst_from.py +++ b/pyemu/utils/pst_from.py @@ -1439,7 +1439,8 @@ def add_parameters( sigma_range: not yet implemented # TODO upper_bound (`float`): PEST parameter upper bound # TODO support different ubound,lbound,transform if multiple use_col lower_bound (`float`): PEST parameter lower bound - transform (`str`): PEST parameter transformation + transform (`str`): PEST parameter transformation. Must be either "log","none" or "fixed. The "tied" transform + must be used after calling `PstFrom.build_pst()`. par_name_base (`str`): basename for parameters that are set up index_cols (`list`-like): if not None, will attempt to parameterize expecting a tabular-style model input file. `index_cols` From 82ec13be46d723679bb9f50ad981987c105ead8c Mon Sep 17 00:00:00 2001 From: unknown Date: Fri, 26 Feb 2021 09:23:28 -0700 Subject: [PATCH 4/7] test for nans in pst dataframes --- autotest/pst_tests.py | 104 ++++++++++++++++++++++++++++++++++++++- autotest/pst_tests_2.py | 2 + pyemu/pst/pst_handler.py | 48 ++++++++++++++---- 3 files changed, 143 insertions(+), 11 deletions(-) diff --git a/autotest/pst_tests.py b/autotest/pst_tests.py index 3dc9afc28..70a023f41 100644 --- a/autotest/pst_tests.py +++ b/autotest/pst_tests.py @@ -957,12 +957,112 @@ def read_in_tpl_test(): assert df.parval1["p5"] == df.parval1["p7"] +def write2_nan_test(): + import numpy as np + import pyemu + import os + + pst = pyemu.Pst(os.path.join("pst", "pest.pst")) + pyemu.helpers.zero_order_tikhonov(pst) + pst.prior_information.loc[pst.prior_names[0], "weight"] = np.NaN + try: + pst.write("test.pst", version=1) + except: + pass + else: + raise Exception("should have failed") + try: + pst.write("test.pst", version=2) + except: + pass + else: + raise Exception("should have failed") + + pst = pyemu.Pst(os.path.join("pst", "pest.pst")) + pst.model_output_data.loc[pst.instruction_files[0], "pest_file"] = np.NaN + try: + pst.write("test.pst", version=1) + except: + pass + else: + raise Exception("should have failed") + try: + pst.write("test.pst", version=2) + except: + pass + else: + raise Exception("should have failed") + + pst = pyemu.Pst(os.path.join("pst", "pest.pst")) + pst.model_input_data.loc[pst.template_files[0], "pest_file"] = np.NaN + try: + pst.write("test.pst", version=1) + except: + pass + else: + raise Exception("should have failed") + try: + pst.write("test.pst", version=2) + except: + pass + else: + raise Exception("should have failed") + + pst = pyemu.Pst(os.path.join("pst","pest.pst")) + pst.parameter_data.loc[pst.par_names[0],"parval1"] = np.NaN + try: + pst.write("test.pst",version=2) + except: + pass + else: + raise Exception("should have failed") + try: + pst.write("test.pst",version=1) + except: + pass + else: + raise Exception("should have failed") + + pst = pyemu.Pst(os.path.join("pst", "pest.pst")) + pst.parameter_groups.loc[pst.parameter_groups.pargpnme[0], "derinc"] = np.NaN + try: + pst.write("test.pst", version=2) + except: + pass + else: + raise Exception("should have failed") + try: + pst.write("test.pst", version=1) + except: + pass + else: + raise Exception("should have failed") + + pst = pyemu.Pst(os.path.join("pst", "pest.pst")) + pst.observation_data.loc[pst.obs_names[0], "weight"] = np.NaN + try: + pst.write("test.pst", version=2) + except: + pass + else: + raise Exception("should have failed") + try: + pst.write("test.pst", version=1) + except: + pass + else: + raise Exception("should have failed") + + + + if __name__ == "__main__": - process_output_files_test() + write2_nan_test() + #process_output_files_test() # change_limit_test() # new_format_test() # lt_gt_constraint_names_test() - csv_to_ins_test() + #csv_to_ins_test() # try_process_ins_test() # write_tables_test() diff --git a/autotest/pst_tests_2.py b/autotest/pst_tests_2.py index 75be1602e..f9f57609b 100644 --- a/autotest/pst_tests_2.py +++ b/autotest/pst_tests_2.py @@ -681,6 +681,8 @@ def at_bounds_test(): assert len(lb) == 1 assert len(ub) == 1 + + if __name__ == "__main__": #at_bounds_test() diff --git a/pyemu/pst/pst_handler.py b/pyemu/pst/pst_handler.py index 1584242b9..d04ab2165 100644 --- a/pyemu/pst/pst_handler.py +++ b/pyemu/pst/pst_handler.py @@ -1530,7 +1530,7 @@ def _write_version2(self, new_filename, use_pst_path=True, pst_rel_path="."): csv_name = "pst.{0}.nans.csv".format( name.replace(" ", "_").replace("*", "") ) - df.to_csv(csv_name) + self.parameter_groups.to_csv(csv_name) raise Exception( "NaNs in {0} dataframe, csv written to {1}".format(name, csv_name) ) @@ -1551,7 +1551,7 @@ def _write_version2(self, new_filename, use_pst_path=True, pst_rel_path="."): csv_name = "pst.{0}.nans.csv".format( name.replace(" ", "_").replace("*", "") ) - df.to_csv(csv_name) + self.parameter_data.to_csv(csv_name) raise Exception( "NaNs in {0} dataframe, csv written to {1}".format(name, csv_name) ) @@ -1572,7 +1572,7 @@ def _write_version2(self, new_filename, use_pst_path=True, pst_rel_path="."): csv_name = "pst.{0}.nans.csv".format( name.replace(" ", "_").replace("*", "") ) - df.to_csv(csv_name) + self.observation_data.to_csv(csv_name) raise Exception( "NaNs in {0} dataframe, csv written to {1}".format(name, csv_name) ) @@ -1596,7 +1596,7 @@ def _write_version2(self, new_filename, use_pst_path=True, pst_rel_path="."): csv_name = "pst.{0}.nans.csv".format( name.replace(" ", "_").replace("*", "") ) - df.to_csv(csv_name) + self.model_input_data.to_csv(csv_name) raise Exception( "NaNs in {0} dataframe, csv written to {1}".format(name, csv_name) ) @@ -1616,7 +1616,7 @@ def _write_version2(self, new_filename, use_pst_path=True, pst_rel_path="."): csv_name = "pst.{0}.nans.csv".format( name.replace(" ", "_").replace("*", "") ) - df.to_csv(csv_name) + self.model_output_data.to_csv(csv_name) raise Exception( "NaNs in {0} dataframe, csv written to {1}".format(name, csv_name) ) @@ -1638,7 +1638,7 @@ def _write_version2(self, new_filename, use_pst_path=True, pst_rel_path="."): csv_name = "pst.{0}.nans.csv".format( name.replace(" ", "_").replace("*", "") ) - df.to_csv(csv_name) + self.prior_information.to_csv(csv_name) raise Exception( "NaNs in {0} dataframe, csv written to {1}".format(name, csv_name) ) @@ -1771,6 +1771,28 @@ def _write_version1(self, new_filename): for cline in self.model_command: f_out.write(cline + "\n") + name = "tplfile_data" + columns = self.model_io_fieldnames + if self.model_input_data.loc[:, columns].isnull().values.any(): + # warnings.warn("WARNING: NaNs in {0} dataframe".format(name)) + csv_name = "pst.{0}.nans.csv".format( + name.replace(" ", "_").replace("*", "") + ) + self.model_input_data.to_csv(csv_name) + raise Exception( + "NaNs in {0} dataframe, csv written to {1}".format(name, csv_name) + ) + name = "insfile_data" + columns = self.model_io_fieldnames + if self.model_output_data.loc[:, columns].isnull().values.any(): + # warnings.warn("WARNING: NaNs in {0} dataframe".format(name)) + csv_name = "pst.{0}.nans.csv".format( + name.replace(" ", "_").replace("*", "") + ) + self.model_output_data.to_csv(csv_name) + raise Exception( + "NaNs in {0} dataframe, csv written to {1}".format(name, csv_name) + ) f_out.write("* model input/output\n") for tplfle, infle in zip( self.model_input_data.pest_file, self.model_input_data.model_file @@ -1782,9 +1804,17 @@ def _write_version1(self, new_filename): f_out.write("{0} {1}\n".format(insfle, outfle)) if self.nprior > 0: - if self.prior_information.isnull().values.any(): - # print("WARNING: NaNs in prior_information dataframe") - warnings.warn("NaNs in prior_information dataframe", PyemuWarning) + name = "pi_data" + columns = self.prior_fieldnames + if self.prior_information.loc[:, columns].isnull().values.any(): + # warnings.warn("WARNING: NaNs in {0} dataframe".format(name)) + csv_name = "pst.{0}.nans.csv".format( + name.replace(" ", "_").replace("*", "") + ) + self.prior_information.to_csv(csv_name) + raise Exception( + "NaNs in {0} dataframe, csv written to {1}".format(name, csv_name) + ) f_out.write("* prior information\n") # self.prior_information.index = self.prior_information.pop("pilbl") max_eq_len = self.prior_information.equation.apply(lambda x: len(x)).max() From 86ffcee0ae0e7c998e4f7fa58cf08ab038ec147e Mon Sep 17 00:00:00 2001 From: unknown Date: Fri, 26 Feb 2021 10:28:01 -0700 Subject: [PATCH 5/7] fix in tpl data trap --- pyemu/pst/pst_handler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyemu/pst/pst_handler.py b/pyemu/pst/pst_handler.py index d04ab2165..467ab68ee 100644 --- a/pyemu/pst/pst_handler.py +++ b/pyemu/pst/pst_handler.py @@ -1601,7 +1601,7 @@ def _write_version2(self, new_filename, use_pst_path=True, pst_rel_path="."): "NaNs in {0} dataframe, csv written to {1}".format(name, csv_name) ) f_out.write("* model input external\n") - io_filename = new_filename.lower().replace(".pst", ".{0}.csv".format(tplfile_data)) + io_filename = new_filename.lower().replace(".pst", ".{0}.csv".format(name)) if pst_path is not None: io_filename = os.path.join(pst_path, os.path.split(io_filename)[-1]) self.model_input_data.to_csv(io_filename, index=False) From abc6304da18cc3ecbf28827675f1f905a0d2abf9 Mon Sep 17 00:00:00 2001 From: unknown Date: Fri, 26 Feb 2021 10:45:41 -0700 Subject: [PATCH 6/7] bug fix in array obs when obsgp is passed --- pyemu/utils/pst_from.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pyemu/utils/pst_from.py b/pyemu/utils/pst_from.py index 7ac55b868..446e110b0 100644 --- a/pyemu/utils/pst_from.py +++ b/pyemu/utils/pst_from.py @@ -1189,7 +1189,9 @@ def add_observations( new_obs = self.add_observations_from_ins( ins_file=insfile, out_file=self.new_d / filename ) - if prefix is not None: + if obsgp is not None: + new_obs.loc[:, "obgnme"] = obsgp + elif prefix is not None: new_obs.loc[:, "obgnme"] = prefix self.logger.log("adding observations from array output file '{0}'".format(filenames)) if rebuild_pst: From 728abf19eee61d038d6ff234c812c5ff8a9f74f2 Mon Sep 17 00:00:00 2001 From: unknown Date: Fri, 26 Feb 2021 15:57:46 -0700 Subject: [PATCH 7/7] small tweak to obs naming without a prefix --- pyemu/pst/pst_utils.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/pyemu/pst/pst_utils.py b/pyemu/pst/pst_utils.py index b68cffa6a..6fa5065b5 100644 --- a/pyemu/pst/pst_utils.py +++ b/pyemu/pst/pst_utils.py @@ -1163,7 +1163,10 @@ def csv_to_ins_file( else: nprefix = prefix if longnames: - nname = f"{nprefix}_usecol:{clabel}" + if len(nprefix) > 0: + nname = f"{nprefix}_usecol:{clabel}" + else: + nname = f"usecol:{clabel}" oname = f"{nname}_{rlabel}" else: nname = nprefix + clabel.replace(" ", "").replace("_", "")