diff --git a/autotest/pst_from_tests.py b/autotest/pst_from_tests.py index 7525f93b8..3801a93cd 100644 --- a/autotest/pst_from_tests.py +++ b/autotest/pst_from_tests.py @@ -2635,7 +2635,7 @@ def mf6_freyberg_arr_obs_test(): #invest() # freyberg_test() #freyberg_prior_build_test() - # mf6_freyberg_test() + mf6_freyberg_test() #mf6_freyberg_shortnames_test() #mf6_freyberg_direct_test() #mf6_freyberg_varying_idomain() @@ -2644,8 +2644,8 @@ def mf6_freyberg_arr_obs_test(): # tpf = TestPstFrom() # tpf.setup() # tpf.test_add_direct_array_parameters() - pstfrom_profile() - mf6_freyberg_arr_obs_test() + #pstfrom_profile() + #mf6_freyberg_arr_obs_test() diff --git a/autotest/pst_tests.py b/autotest/pst_tests.py index 02d0c93fe..eedaa2d3c 100644 --- a/autotest/pst_tests.py +++ b/autotest/pst_tests.py @@ -945,6 +945,17 @@ def ctrl_data_test(): pst2.write(os.path.join("pst","test2.pst"),version=2) pst3 = pyemu.Pst(os.path.join("pst", "test2.pst")) +def read_in_tpl_test(): + import pyemu + tpl_d = "tpl" + tpl_files = [os.path.join(tpl_d,f) for f in os.listdir(tpl_d) if f.endswith(".tpl")] + df = pyemu.pst_utils.try_read_input_file_with_tpl(os.path.join(tpl_d,"test1.dat.tpl")) + print(df) + assert df.parval1["p1"] == df.parval1["p2"] + assert df.parval1["p3"] == df.parval1["p4"] + assert df.parval1["p5"] == df.parval1["p6"] + assert df.parval1["p5"] == df.parval1["p7"] + if __name__ == "__main__": process_output_files_test() @@ -990,5 +1001,6 @@ def ctrl_data_test(): #try_process_ins_test() #tpl_ins_test() #process_output_files_test() + #comments_test() + read_in_tpl_test() comments_test() - \ No newline at end of file diff --git a/autotest/pst_tests_2.py b/autotest/pst_tests_2.py index 29fbf7b00..75be1602e 100644 --- a/autotest/pst_tests_2.py +++ b/autotest/pst_tests_2.py @@ -522,6 +522,7 @@ def pst_from_flopy_geo_draw_test(): def from_flopy_pp_test(): import numpy as np + import pandas as pd try: import flopy except: @@ -547,6 +548,23 @@ def from_flopy_pp_test(): use_pp_zones=False, build_prior=False) + b_d = os.getcwd() + os.chdir(new_model_ws) + try: + pyemu.helpers.apply_array_pars() + except Exception as e: + os.chdir(b_d) + raise (str(e)) + os.chdir(b_d) + + + mlt_dir = os.path.join(new_model_ws,"arr_mlt") + for f in os.listdir(mlt_dir): + arr = np.loadtxt(os.path.join(mlt_dir,f)) + assert np.all(arr==1) + df = pd.read_csv(os.path.join(new_model_ws, "arr_pars.csv"), index_col=0) + assert np.all(df.pp_fill_value.values == 1) + new_model_ws = "temp_pst_from_flopy" props = ["upw.ss","upw.hk","upw.vka"] pp_props = [] @@ -564,6 +582,7 @@ def from_flopy_pp_test(): build_prior=True) + def pst_from_flopy_specsim_draw_test(): import shutil import numpy as np @@ -670,10 +689,10 @@ def at_bounds_test(): #pst_from_flopy_specsim_draw_test() # run_array_pars() # from_flopy_zone_pars() - # from_flopy_pp_test() + #from_flopy_pp_test() # from_flopy() - # from_flopy_kl_test() - from_flopy_reachinput() + from_flopy_kl_test() + #from_flopy_reachinput() diff --git a/autotest/tpl/test1.dat b/autotest/tpl/test1.dat new file mode 100644 index 000000000..8f3d3da9d --- /dev/null +++ b/autotest/tpl/test1.dat @@ -0,0 +1,4 @@ + +1 1.2345678 1.2345678 1.2345678 9.87654321 +222222222222222222222222222222222222 +12.312.312.3 \ No newline at end of file diff --git a/autotest/tpl/test1.dat.tpl b/autotest/tpl/test1.dat.tpl new file mode 100644 index 000000000..77dd3badb --- /dev/null +++ b/autotest/tpl/test1.dat.tpl @@ -0,0 +1,5 @@ +ptf ~ + +1 ~ p1 ~ ~ p1 ~ ~ p2 ~ ~ p9 ~ +1 ~ p3 ~ ~ p4 ~ +~p5~~p6~~p7~ \ No newline at end of file diff --git a/pyemu/pst/pst_handler.py b/pyemu/pst/pst_handler.py index b15d46960..275fd4600 100644 --- a/pyemu/pst/pst_handler.py +++ b/pyemu/pst/pst_handler.py @@ -2452,6 +2452,9 @@ def add_parameters(self, template_file, in_file=None, pst_path=None): # get the parameter names in the template file parnme = pst_utils.parse_tpl_file(template_file) + parval1 = pst_utils.try_read_input_file_with_tpl(template_file,in_file) + + # find "new" parameters that are not already in the control file new_parnme = [p for p in parnme if p not in self.parameter_data.parnme] @@ -2472,6 +2475,8 @@ def add_parameters(self, template_file, in_file=None, pst_path=None): ) new_par_data.loc[new_parnme, "parnme"] = new_parnme self.parameter_data = self.parameter_data.append(new_par_data) + if parval1 is not None: + new_par_data.loc[parval1.parnme,"parval1"] = parval1.parval1 if in_file is None: in_file = template_file.replace(".tpl", "") if pst_path is not None: diff --git a/pyemu/pst/pst_utils.py b/pyemu/pst/pst_utils.py index 20622018d..d624f431d 100644 --- a/pyemu/pst/pst_utils.py +++ b/pyemu/pst/pst_utils.py @@ -719,6 +719,109 @@ def generic_pst(par_names=["par1"], obs_names=["obs1"], addreg=False): return new_pst +def try_read_input_file_with_tpl(tpl_file,input_file=None): + """attempt to read parameter values from an input file using a template file + Args: + tpl_file (`str`): path and name of a template file + input_file (`str`,optional): path and name of existing model + input file to process. If `None`, `tpl_file.replace(".tpl","")` + is used. Default is None. + + Returns: + `pandas.DataFrame`: a dataframe of parameter name and values + extracted from `input_file`. + + Note: + If an exception is raised when reading the input file, the exception + is echoed to the screen and `None` is returned. + + Example:: + + df = pyemu.pst_utils.try_process_output_file("my.tpl","my.input") + + """ + + if input_file is None: + input_file = tpl_file.replace(".tpl", "") + if not os.path.exists(input_file): + return None + # read the names first to see what we are dealing with + # and also to do some basic error checking + parnames = parse_tpl_file(tpl_file) + try: + df = _read_infile_with_tplfile(tpl_file,input_file) + except Exception as e: + print("error trying to read input file with tpl file:{0}".format(str(e))) + return None + return df + +def _read_infile_with_tplfile(tpl_file,input_file): + """attempt to read parameter values from an input file using a template file, + raising heaps of exceptions. + Args: + tpl_file (`str`): path and name of a template file + input_file (`str`): path and name of existing model + + Returns: + `pandas.DataFrame`: a dataframe of parameter name and values + extracted from `input_file`. + + Note: + use try_read_inputfile_with_tpl instead of this one. + + """ + + if not os.path.exists(input_file): + raise Exception("input file '{0}' not found".format(input_file)) + + f_tpl = open(tpl_file,'r') + f_in = open(input_file,'r') + + # read the tpl header + _, marker = f_tpl.readline().split() + itpl,iin = 1,0 + pnames,pvals = [],[] + pdict = {} + while True: + tpl_line =f_tpl.readline() + if tpl_line == "": + break + + in_line = f_in.readline() + if in_line == "": + raise Exception("input file EOF, tpl file line {0}, in file line {1}".format(itpl,iin)) + + if marker in tpl_line: + idxs = [i for i, ltr in enumerate(tpl_line) if ltr == marker] + if len(idxs) % 2 != 0: + raise Exception("unbalanced markers on tpl line {0}".format(itpl)) + + for s,e in zip(idxs[0:-1:2],idxs[1::2]): + tpl_str = tpl_line[s:e] + pname = tpl_str.replace(marker,"").strip().lower() + if s > len(in_line): + raise Exception("input file EOL line {0}, tpl line {1}, looking for {2}"\ + .format(iin,itpl,tpl_str)) + in_str = in_line[s:e] + try: + v = float(in_str) + except Exception as e: + raise Exception("error casting '{0}' to float on in line {1}, tpl line {2} for {3}: {4}".\ + format(in_str,iin,itpl,tpl_str,str(e))) + + if pname in pdict: + eval = pdict[pname] + if not np.isclose(eval,v,1.0e-6): + raise Exception("different values {0}:{1} for par {2} on in line {3}".format(v,eval,pname,iin)) + else: + pnames.append(pname) + pvals.append(v) + pdict[pname] = v + itpl += 1 + iin += 1 + df = pd.DataFrame({"parnme":pnames,"parval1":pvals},index=pnames) + return df + def try_process_output_file(ins_file, output_file=None): """attempt to process a model output file using a PEST-style instruction file diff --git a/pyemu/utils/helpers.py b/pyemu/utils/helpers.py index 606d7d998..10544d93e 100644 --- a/pyemu/utils/helpers.py +++ b/pyemu/utils/helpers.py @@ -2343,6 +2343,10 @@ def _pp_prep(self, mlt_df): pp_file = pp_files.iloc[0] mlt_df.loc[mlt_df.mlt_file == out_file, "fac_file"] = fac_file mlt_df.loc[mlt_df.mlt_file == out_file, "pp_file"] = pp_file + mlt_df.loc[mlt_df.mlt_file == out_file, "pp_fill_value"] = 1.0 + mlt_df.loc[mlt_df.mlt_file == out_file, "pp_lower_limit"] = 1.0e-10 + mlt_df.loc[mlt_df.mlt_file == out_file, "pp_upper_limit"] = 1.0e+10 + self.par_dfs[self.pp_suffix] = pp_df mlt_df.loc[mlt_df.suffix == self.pp_suffix, "tpl_file"] = np.NaN @@ -2408,6 +2412,9 @@ def _kl_prep(self, mlt_df): mlt_df.loc[mlt_df.prefix == prefix, "fac_file"] = os.path.split(fac_file)[ -1 ] + mlt_df.loc[mlt_df.prefix == prefix, "pp_fill_value"] = 1.0 + mlt_df.loc[mlt_df.prefix == prefix, "pp_lower_limit"] = 1.0e-10 + mlt_df.loc[mlt_df.prefix == prefix, "pp_upper_limit"] = 1.0e+10 print(kl_mlt_df) mlt_df.loc[mlt_df.suffix == self.kl_suffix, "tpl_file"] = np.NaN @@ -3702,10 +3709,11 @@ def apply_array_pars(arr_par="arr_pars.csv", arr_par_file=None, chunk_len=50): if "pp_file" in df.columns: print("starting fac2real", datetime.now()) - pp_df = df.loc[df.pp_file.notna(), ["pp_file", "fac_file", "mlt_file"]].rename( - columns={"fac_file": "factors_file", "mlt_file": "out_file"} + pp_df = df.loc[df.pp_file.notna(), ["pp_file", "fac_file", "mlt_file", + "pp_fill_value","pp_lower_limit","pp_upper_limit"]].rename( + columns={"fac_file": "factors_file", "mlt_file": "out_file", + "pp_fill_value":"fill_value","pp_lower_limit":"lower_lim","pp_upper_limit":"upper_lim"} ) - pp_df.loc[:, "lower_lim"] = 1.0e-10 # don't need to process all (e.g. if const. mults apply across kper...) pp_args = pp_df.drop_duplicates().to_dict("records") num_ppargs = len(pp_args) diff --git a/pyemu/utils/pst_from.py b/pyemu/utils/pst_from.py index f87ad9013..c1cc8bda9 100644 --- a/pyemu/utils/pst_from.py +++ b/pyemu/utils/pst_from.py @@ -1075,7 +1075,7 @@ def _process_array_obs(self,out_filename,ins_filename,prefix,ofile_sep,ofile_ski if zval is not None: oname += "_zone:{0}".format(zval) else: - oname = "{0}_{1}_{2}".format(prefix,iline,jr) + oname = "{0}_{1}_{2}".format(prefix,iidx,jr) if zval is not None: z_str = "_{0}".format(zval) if len(oname) + len(z_str) < 20: @@ -1537,6 +1537,8 @@ def add_parameters( filenames = [ get_relative_filepath(self.original_d, filename) for filename in filenames ] + if len(filenames) == 0: + self.logger.lraise("add_parameters(): filenames is empty") if par_style == "direct": if len(filenames) != 1: self.logger.lraise( @@ -1925,7 +1927,7 @@ def add_parameters( pp_info_dict = { "pp_data": ok_pp.point_data.loc[:, ["x", "y", "zone"]], "cov": ok_pp.point_cov_df, - "zn_ar": zone_array, + "zn_ar": zone_array } fac_processed = False for facfile, info in self._pp_facs.items(): # check against @@ -2004,6 +2006,9 @@ def add_parameters( assert fac_filename is not None, "missing pilot-point input filename" mult_dict["fac_file"] = os.path.relpath(fac_filename, self.new_d) mult_dict["pp_file"] = pp_filename + mult_dict["pp_fill_value"] = 1.0 + mult_dict["pp_lower_limit"] = 1.0e-10 + mult_dict["pp_upper_limit"] = 1.0e+10 relate_parfiles.append(mult_dict) relate_pars_df = pd.DataFrame(relate_parfiles) # store on self for use in pest build etc