From 1395b97f55accd9b6de8032fc782fdabd691257c Mon Sep 17 00:00:00 2001 From: Jeremy White Date: Fri, 8 Jan 2021 14:57:10 -0700 Subject: [PATCH 1/5] added input file reading with tpl...maybe this is a good idea? --- autotest/pst_tests.py | 16 +++++- autotest/tpl/test1.dat | 4 ++ autotest/tpl/test1.dat.tpl | 5 ++ pyemu/pst/pst_handler.py | 5 ++ pyemu/pst/pst_utils.py | 103 +++++++++++++++++++++++++++++++++++++ 5 files changed, 132 insertions(+), 1 deletion(-) create mode 100644 autotest/tpl/test1.dat create mode 100644 autotest/tpl/test1.dat.tpl diff --git a/autotest/pst_tests.py b/autotest/pst_tests.py index a51d6101a..bdadb9d80 100644 --- a/autotest/pst_tests.py +++ b/autotest/pst_tests.py @@ -945,6 +945,19 @@ def ctrl_data_test(): pst2.write(os.path.join("pst","test2.pst"),version=2) pst3 = pyemu.Pst(os.path.join("pst", "test2.pst")) + +def read_in_tpl_test(): + import pyemu + tpl_d = "tpl" + tpl_files = [os.path.join(tpl_d,f) for f in os.listdir(tpl_d) if f.endswith(".tpl")] + df = pyemu.pst_utils.try_read_input_file_with_tpl(os.path.join(tpl_d,"test1.dat.tpl")) + print(df) + assert df.parval1["p1"] == df.parval1["p2"] + assert df.parval1["p3"] == df.parval1["p4"] + assert df.parval1["p5"] == df.parval1["p6"] + assert df.parval1["p5"] == df.parval1["p7"] + + if __name__ == "__main__": # process_output_files_test() @@ -990,4 +1003,5 @@ def ctrl_data_test(): #try_process_ins_test() #tpl_ins_test() #process_output_files_test() - comments_test() \ No newline at end of file + #comments_test() + read_in_tpl_test() \ No newline at end of file diff --git a/autotest/tpl/test1.dat b/autotest/tpl/test1.dat new file mode 100644 index 000000000..8f3d3da9d --- /dev/null +++ b/autotest/tpl/test1.dat @@ -0,0 +1,4 @@ + +1 1.2345678 1.2345678 1.2345678 9.87654321 +222222222222222222222222222222222222 +12.312.312.3 \ No newline at end of file diff --git a/autotest/tpl/test1.dat.tpl b/autotest/tpl/test1.dat.tpl new file mode 100644 index 000000000..77dd3badb --- /dev/null +++ b/autotest/tpl/test1.dat.tpl @@ -0,0 +1,5 @@ +ptf ~ + +1 ~ p1 ~ ~ p1 ~ ~ p2 ~ ~ p9 ~ +1 ~ p3 ~ ~ p4 ~ +~p5~~p6~~p7~ \ No newline at end of file diff --git a/pyemu/pst/pst_handler.py b/pyemu/pst/pst_handler.py index b15d46960..275fd4600 100644 --- a/pyemu/pst/pst_handler.py +++ b/pyemu/pst/pst_handler.py @@ -2452,6 +2452,9 @@ def add_parameters(self, template_file, in_file=None, pst_path=None): # get the parameter names in the template file parnme = pst_utils.parse_tpl_file(template_file) + parval1 = pst_utils.try_read_input_file_with_tpl(template_file,in_file) + + # find "new" parameters that are not already in the control file new_parnme = [p for p in parnme if p not in self.parameter_data.parnme] @@ -2472,6 +2475,8 @@ def add_parameters(self, template_file, in_file=None, pst_path=None): ) new_par_data.loc[new_parnme, "parnme"] = new_parnme self.parameter_data = self.parameter_data.append(new_par_data) + if parval1 is not None: + new_par_data.loc[parval1.parnme,"parval1"] = parval1.parval1 if in_file is None: in_file = template_file.replace(".tpl", "") if pst_path is not None: diff --git a/pyemu/pst/pst_utils.py b/pyemu/pst/pst_utils.py index 5aa37c3d9..73a8fb94d 100644 --- a/pyemu/pst/pst_utils.py +++ b/pyemu/pst/pst_utils.py @@ -715,6 +715,109 @@ def generic_pst(par_names=["par1"], obs_names=["obs1"], addreg=False): return new_pst +def try_read_input_file_with_tpl(tpl_file,input_file=None): + """attempt to read parameter values from an input file using a template file + Args: + tpl_file (`str`): path and name of a template file + input_file (`str`,optional): path and name of existing model + input file to process. If `None`, `tpl_file.replace(".tpl","")` + is used. Default is None. + + Returns: + `pandas.DataFrame`: a dataframe of parameter name and values + extracted from `input_file`. + + Note: + If an exception is raised when reading the input file, the exception + is echoed to the screen and `None` is returned. + + Example:: + + df = pyemu.pst_utils.try_process_output_file("my.tpl","my.input") + + """ + + if input_file is None: + input_file = tpl_file.replace(".tpl", "") + if not os.path.exists(input_file): + return None + # read the names first to see what we are dealing with + # and also to do some basic error checking + parnames = parse_tpl_file(tpl_file) + try: + df = _read_infile_with_tplfile(tpl_file,input_file) + except Exception as e: + print("error trying to read input file with tpl file:{0}".format(str(e))) + return None + return df + +def _read_infile_with_tplfile(tpl_file,input_file): + """attempt to read parameter values from an input file using a template file, + raising heaps of exceptions. + Args: + tpl_file (`str`): path and name of a template file + input_file (`str`): path and name of existing model + + Returns: + `pandas.DataFrame`: a dataframe of parameter name and values + extracted from `input_file`. + + Note: + use try_read_inputfile_with_tpl instead of this one. + + """ + + if not os.path.exists(input_file): + raise Exception("input file '{0}' not found".format(input_file)) + + f_tpl = open(tpl_file,'r') + f_in = open(input_file,'r') + + # read the tpl header + _, marker = f_tpl.readline().split() + itpl,iin = 1,0 + pnames,pvals = [],[] + pdict = {} + while True: + tpl_line =f_tpl.readline() + if tpl_line == "": + break + + in_line = f_in.readline() + if in_line == "": + raise Exception("input file EOF, tpl file line {0}, in file line {1}".format(itpl,iin)) + + if marker in tpl_line: + idxs = [i for i, ltr in enumerate(tpl_line) if ltr == marker] + if len(idxs) % 2 != 0: + raise Exception("unbalanced markers on tpl line {0}".format(itpl)) + + for s,e in zip(idxs[0:-1:2],idxs[1::2]): + tpl_str = tpl_line[s:e] + pname = tpl_str.replace(marker,"").strip().lower() + if s > len(in_line): + raise Exception("input file EOL line {0}, tpl line {1}, looking for {2}"\ + .format(iin,itpl,tpl_str)) + in_str = in_line[s:e] + try: + v = float(in_str) + except Exception as e: + raise Exception("error casting '{0}' to float on in line {1}, tpl line {2} for {3}: {4}".\ + format(in_str,iin,itpl,tpl_str,str(e))) + + if pname in pdict: + eval = pdict[pname] + if not np.isclose(eval,v,1.0e-6): + raise Exception("different values {0}:{1} for par {2} on in line {3}".format(v,eval,pname,iin)) + else: + pnames.append(pname) + pvals.append(v) + pdict[pname] = v + itpl += 1 + iin += 1 + df = pd.DataFrame({"parnme":pnames,"parval1":pvals},index=pnames) + return df + def try_process_output_file(ins_file, output_file=None): """attempt to process a model output file using a PEST-style instruction file From 9af911b3656d72d327def1b4e41bebb4a3b23f7c Mon Sep 17 00:00:00 2001 From: unknown Date: Sat, 9 Jan 2021 14:16:42 -0700 Subject: [PATCH 2/5] trap for empty filenames in add_pars --- pyemu/utils/pst_from.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pyemu/utils/pst_from.py b/pyemu/utils/pst_from.py index d2780e85e..a118e0586 100644 --- a/pyemu/utils/pst_from.py +++ b/pyemu/utils/pst_from.py @@ -1537,6 +1537,8 @@ def add_parameters( filenames = [ get_relative_filepath(self.original_d, filename) for filename in filenames ] + if len(filenames) == 0: + self.logger.lraise("add_parameters(): filenames is empty") if par_style == "direct": if len(filenames) != 1: self.logger.lraise( From 248d97d80daa373768b5470d5d364008a94913d9 Mon Sep 17 00:00:00 2001 From: Jeremy White Date: Mon, 11 Jan 2021 10:36:08 -0700 Subject: [PATCH 3/5] fix for kl pars --- autotest/pst_tests_2.py | 4 ++-- pyemu/utils/helpers.py | 3 +++ 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/autotest/pst_tests_2.py b/autotest/pst_tests_2.py index ad25a445f..75be1602e 100644 --- a/autotest/pst_tests_2.py +++ b/autotest/pst_tests_2.py @@ -689,10 +689,10 @@ def at_bounds_test(): #pst_from_flopy_specsim_draw_test() # run_array_pars() # from_flopy_zone_pars() - from_flopy_pp_test() + #from_flopy_pp_test() # from_flopy() - # from_flopy_kl_test() + from_flopy_kl_test() #from_flopy_reachinput() diff --git a/pyemu/utils/helpers.py b/pyemu/utils/helpers.py index 4aee34db2..10544d93e 100644 --- a/pyemu/utils/helpers.py +++ b/pyemu/utils/helpers.py @@ -2412,6 +2412,9 @@ def _kl_prep(self, mlt_df): mlt_df.loc[mlt_df.prefix == prefix, "fac_file"] = os.path.split(fac_file)[ -1 ] + mlt_df.loc[mlt_df.prefix == prefix, "pp_fill_value"] = 1.0 + mlt_df.loc[mlt_df.prefix == prefix, "pp_lower_limit"] = 1.0e-10 + mlt_df.loc[mlt_df.prefix == prefix, "pp_upper_limit"] = 1.0e+10 print(kl_mlt_df) mlt_df.loc[mlt_df.suffix == self.kl_suffix, "tpl_file"] = np.NaN From f20fb89248ef6622a221ed4377e65d3aa87eb2e8 Mon Sep 17 00:00:00 2001 From: unknown Date: Thu, 14 Jan 2021 11:46:24 -0700 Subject: [PATCH 4/5] fix in arr obs i index tracking --- pyemu/utils/pst_from.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyemu/utils/pst_from.py b/pyemu/utils/pst_from.py index d9cd90e56..4809c607f 100644 --- a/pyemu/utils/pst_from.py +++ b/pyemu/utils/pst_from.py @@ -1071,7 +1071,7 @@ def _process_array_obs(self,out_filename,ins_filename,prefix,ofile_sep,ofile_ski continue if longnames: - oname = "arrobs_{0}_i:{1}_j:{2}".format(prefix,iidx,jr) + oname = "arrobs_{0}_i:{1}_j:{2}".format(prefix,iline,jr) if zval is not None: oname += "_zone:{0}".format(zval) else: From 382fe6a2aa1d1d8842339b5861f1038557c740c6 Mon Sep 17 00:00:00 2001 From: unknown Date: Thu, 14 Jan 2021 12:59:25 -0700 Subject: [PATCH 5/5] index issue in array obs --- pyemu/utils/pst_from.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyemu/utils/pst_from.py b/pyemu/utils/pst_from.py index 4809c607f..c1cc8bda9 100644 --- a/pyemu/utils/pst_from.py +++ b/pyemu/utils/pst_from.py @@ -1071,11 +1071,11 @@ def _process_array_obs(self,out_filename,ins_filename,prefix,ofile_sep,ofile_ski continue if longnames: - oname = "arrobs_{0}_i:{1}_j:{2}".format(prefix,iline,jr) + oname = "arrobs_{0}_i:{1}_j:{2}".format(prefix,iidx,jr) if zval is not None: oname += "_zone:{0}".format(zval) else: - oname = "{0}_{1}_{2}".format(prefix,iline,jr) + oname = "{0}_{1}_{2}".format(prefix,iidx,jr) if zval is not None: z_str = "_{0}".format(zval) if len(oname) + len(z_str) < 20: