From 922ac5c453510277a5e8173b4302db1cfa1dc8c3 Mon Sep 17 00:00:00 2001 From: White Date: Tue, 18 Aug 2020 13:26:51 -0600 Subject: [PATCH 1/7] casting user-supplied pargp and par_base_name to lower, otherwise, bad times --- pyemu/utils/pst_from.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/pyemu/utils/pst_from.py b/pyemu/utils/pst_from.py index 9cf7a1621..0cd414f09 100644 --- a/pyemu/utils/pst_from.py +++ b/pyemu/utils/pst_from.py @@ -851,6 +851,7 @@ def add_observations(self, filename, insfile=None, use_rows = df.iloc[use_rows].idx_str.unique() # construct ins_file from df ncol = len(use_cols) + obsgp = _check_var_len(obsgp, ncol, fill=True) df_ins = pyemu.pst_utils.csv_to_ins_file( df.set_index('idx_str'), @@ -1159,6 +1160,12 @@ def add_parameters(self, filenames, par_type, zone_array=None, "single-element container, or container of " "len use_cols, not '{0}'" "".format(str(par_name_base))) + + # otherewise, things get tripped up in the ensemble/cov stuff + if pargp is not None: + pargp = pargp.lower() + par_name_base = [pnb.lower() for pnb in par_name_base] + if self.longnames: # allow par names to be long... fine for pestpp fmt = "_{0}".format(alt_inst_str) + ":{0}" chk_prefix = "_{0}".format(alt_inst_str) # add `instance` identifier @@ -1169,9 +1176,11 @@ def add_parameters(self, filenames, par_type, zone_array=None, for i in range(len(par_name_base)): par_name_base[i] += fmt.format( self._next_count(par_name_base[i] + chk_prefix)) + # multiplier file name will be taken first par group, if passed # (the same multipliers will apply to all pars passed in this call) # Remove `:` for filenames + par_name_store = par_name_base[0].replace(':', '') # for os filename # Define requisite filenames From 9dcaf9fe7e425d300b719b1903a31f6c0648757c Mon Sep 17 00:00:00 2001 From: White Date: Tue, 18 Aug 2020 14:26:35 -0600 Subject: [PATCH 2/7] casting user-supplied pargp and par_base_name to lower, otherwise, bad times --- pyemu/utils/pst_from.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/pyemu/utils/pst_from.py b/pyemu/utils/pst_from.py index 0cd414f09..5705df3f7 100644 --- a/pyemu/utils/pst_from.py +++ b/pyemu/utils/pst_from.py @@ -1163,7 +1163,10 @@ def add_parameters(self, filenames, par_type, zone_array=None, # otherewise, things get tripped up in the ensemble/cov stuff if pargp is not None: - pargp = pargp.lower() + if isinstance(pargp,list): + pargp = [pg.lower() for pg in pargp] + else: + pargp = pargp.lower() par_name_base = [pnb.lower() for pnb in par_name_base] if self.longnames: # allow par names to be long... fine for pestpp From 798902661c953bac4993d431136e905ddfb083ea Mon Sep 17 00:00:00 2001 From: White Date: Tue, 18 Aug 2020 18:26:39 -0600 Subject: [PATCH 3/7] added default ult upper and lower bounds to try to prevent denormal numbers --- pyemu/utils/pst_from.py | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/pyemu/utils/pst_from.py b/pyemu/utils/pst_from.py index 5705df3f7..58af04d52 100644 --- a/pyemu/utils/pst_from.py +++ b/pyemu/utils/pst_from.py @@ -1049,9 +1049,11 @@ def add_parameters(self, filenames, par_type, zone_array=None, when reading and reapply when writing. Can optionally be `str` in which case `mf_skip` will be treated as a `comment_char`. ult_ubound (`float`): Ultimate upper bound for model input - parameter once all mults are applied - ensure physical model par vals + parameter once all mults are applied - ensure physical model par vals. If not passed, + it is set to 1.0e+30 ult_lbound (`float`): Ultimate lower bound for model input - parameter once all mults are applied + parameter once all mults are applied. If not passed, it is set to + 1.0e-30 for log transform and -1.0e+30 for non-log transform rebuild_pst (`bool`): (Re)Construct PstFrom.pst object after adding new parameters alt_inst_str (`str`): Alternative to default `inst` string in @@ -1071,6 +1073,16 @@ def add_parameters(self, filenames, par_type, zone_array=None, # TODO support passing par_file (i,j)/(x,y) directly where information # is not contained in model parameter file - e.g. no i,j columns + + # this keeps denormal values for creeping into the model input arrays + if ult_ubound is None: + ult_ubound = 1.0e+30 + if ult_lbound is None: + if transform.lower() == "log": + ult_lbound == 1.0e-30 + else: + ult_lbound = -1.0e+30 + #some checks for direct parameters par_style = par_style.lower() if par_style not in ["multiplier", "direct"]: From 1a13d1d04f127a31edb56fe0b9a0110e648b6f54 Mon Sep 17 00:00:00 2001 From: White Date: Wed, 19 Aug 2020 10:09:40 -0500 Subject: [PATCH 4/7] simplified ult bound setup, added default ult bound values, support for multiple ult bound values: using min upper and max lower --- autotest/pst_from_tests.py | 12 ++++----- pyemu/utils/pst_from.py | 50 +++++++++----------------------------- 2 files changed, 18 insertions(+), 44 deletions(-) diff --git a/autotest/pst_from_tests.py b/autotest/pst_from_tests.py index d7dc0b990..2a5c358dd 100644 --- a/autotest/pst_from_tests.py +++ b/autotest/pst_from_tests.py @@ -210,12 +210,12 @@ def freyberg_test(): # test par mults are working b_d = os.getcwd() os.chdir(pf.new_d) - try: - pyemu.helpers.apply_list_and_array_pars( - arr_par_file="mult2model_info.csv") - except Exception as e: - os.chdir(b_d) - raise Exception(str(e)) + #try: + pyemu.helpers.apply_list_and_array_pars( + arr_par_file="mult2model_info.csv") + # except Exception as e: + # os.chdir(b_d) + # raise Exception(str(e)) os.chdir(b_d) pst.control_data.noptmax = 0 diff --git a/pyemu/utils/pst_from.py b/pyemu/utils/pst_from.py index 58af04d52..a475b1a06 100644 --- a/pyemu/utils/pst_from.py +++ b/pyemu/utils/pst_from.py @@ -122,6 +122,8 @@ def __init__(self, original_d, new_d, longnames=True, self.pst = None self._function_lines_list = [] #each function is itself a list of lines self.direct_org_files = [] + self.ult_ubound_fill = 1.0e+30 + self.ult_lbound_fill = -1.0e+30 @property def parfile_relations(self): @@ -150,25 +152,10 @@ def parfile_relations(self): else pd.Series( {k: v for k, v in [['ubound', x.upper_bound]]}), axis=1) if ubound.nunique(0, False).gt(1).any(): - if ubound.nunique(0, False).gt(2).any(): - # more than one upper bound set - self.logger.lraise( - "different upper bounds requested for same par for {0}" - "".format(name)) - else: - # one set - the rest are None - need to replace None - # with set values - # df with set values - fil = ubound.apply(lambda x: - pd.Series([None]) if x.isna().all() - else x[x.notna()].values).T - self.logger.warn("Upper bound for par passed for some but " - "not all instances, will set NA to " - "passed values\n{}".format(fil)) - # replace Nones in list in Series with passed values - pr.loc[g.index, 'upper_bound'] = g.use_cols.apply( - lambda x: [fil[0].loc['ubound{0}'.format(c)] for c in x] - if x is not None else fil[0].loc['ubound']) + ub_min = ubound.min().fillna(self.ult_ubound_fill).to_dict() + pr.loc[g.index, 'upper_bound'] = g.use_cols.apply( + lambda x: [ub_min['ubound{0}'.format(c)] for c in x] + if x is not None else ub_min["ubound"]) # repeat for lower bounds lbound = g.apply( lambda x: pd.Series( @@ -178,20 +165,10 @@ def parfile_relations(self): else pd.Series( {k: v for k, v in [['lbound', x.lower_bound]]}), axis=1) if lbound.nunique(0, False).gt(1).any(): - if lbound.nunique(0, False).gt(2).any(): - self.logger.lraise( - "different lower bounds requested for same par for {0}" - "".format(name)) - else: - fil = lbound.apply(lambda x: - pd.Series([None]) if x.isna().all() - else x[x.notna()].values).T - self.logger.warn("Lower bound for par passed for some but " - "not all instances, will set NA to " - "passed values\n{}".format(fil)) - pr.loc[g.index, 'lower_bound'] = g.use_cols.apply( - lambda x: [fil[0].loc['lbound{0}'.format(c)] for c in x] - if x is not None else fil[0].loc['lbound']) + lb_max = lbound.min().fillna(self.ult_lbound_fill).to_dict() + pr.loc[g.index, 'lower_bound'] = g.use_cols.apply( + lambda x: [lb_max['lbound{0}'.format(c)] for c in x] + if x is not None else lb_max['lbound']) pr['zero_based'] = self.zero_based return pr @@ -1076,12 +1053,9 @@ def add_parameters(self, filenames, par_type, zone_array=None, # this keeps denormal values for creeping into the model input arrays if ult_ubound is None: - ult_ubound = 1.0e+30 + ult_ubound = self.ult_ubound_fill if ult_lbound is None: - if transform.lower() == "log": - ult_lbound == 1.0e-30 - else: - ult_lbound = -1.0e+30 + ult_lbound = self.ult_lbound_fill #some checks for direct parameters par_style = par_style.lower() From e570332aed46263d5c9d2419d016ea023645d1d0 Mon Sep 17 00:00:00 2001 From: White Date: Thu, 20 Aug 2020 09:36:10 -0500 Subject: [PATCH 5/7] undo error trap in psfrom test --- autotest/pst_from_tests.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/autotest/pst_from_tests.py b/autotest/pst_from_tests.py index 2a5c358dd..d7dc0b990 100644 --- a/autotest/pst_from_tests.py +++ b/autotest/pst_from_tests.py @@ -210,12 +210,12 @@ def freyberg_test(): # test par mults are working b_d = os.getcwd() os.chdir(pf.new_d) - #try: - pyemu.helpers.apply_list_and_array_pars( - arr_par_file="mult2model_info.csv") - # except Exception as e: - # os.chdir(b_d) - # raise Exception(str(e)) + try: + pyemu.helpers.apply_list_and_array_pars( + arr_par_file="mult2model_info.csv") + except Exception as e: + os.chdir(b_d) + raise Exception(str(e)) os.chdir(b_d) pst.control_data.noptmax = 0 From feff18bf6470d46a88d6ca8b40b3f13d6c178d96 Mon Sep 17 00:00:00 2001 From: White Date: Thu, 20 Aug 2020 10:57:07 -0500 Subject: [PATCH 6/7] swapped ult lower bound to max, added test for ult bounds --- autotest/pst_from_tests.py | 32 +++++++++++++++++++++++++------- pyemu/utils/pst_from.py | 2 +- 2 files changed, 26 insertions(+), 8 deletions(-) diff --git a/autotest/pst_from_tests.py b/autotest/pst_from_tests.py index d7dc0b990..6b44d4d76 100644 --- a/autotest/pst_from_tests.py +++ b/autotest/pst_from_tests.py @@ -509,7 +509,7 @@ def mf6_freyberg_test(): sim.simulation_data.mfpath.set_sim_path(tmp_model_ws) # sim.set_all_data_external() m = sim.get_model("freyberg6") - sim.set_all_data_external() + sim.set_all_data_external(check_data=False) sim.write_simulation() # to by pass the issues with flopy @@ -633,11 +633,22 @@ def mf6_freyberg_test(): datetime=dts[kper]) else: for arr_file in arr_files: + + # these ult bounds are used later in an assert + ult_lb = None + ult_ub = None + if "npf_k_" in arr_file: + ult_ub = 20.0 + ult_lb = 2.0 pf.add_parameters(filenames=arr_file,par_type="grid",par_name_base=arr_file.split('.')[1]+"_gr", pargp=arr_file.split('.')[1]+"_gr",zone_array=ib,upper_bound=ub,lower_bound=lb, - geostruct=gr_gs) + geostruct=gr_gs,ult_ubound=None if ult_ub is None else ult_ub + 1, + ult_lbound=None if ult_lb is None else ult_lb + 1) + # use a slightly lower ult bound here pf.add_parameters(filenames=arr_file, par_type="pilotpoints", par_name_base=arr_file.split('.')[1]+"_pp", - pargp=arr_file.split('.')[1]+"_pp", zone_array=ib,upper_bound=ub,lower_bound=lb,) + pargp=arr_file.split('.')[1]+"_pp", zone_array=ib,upper_bound=ub,lower_bound=lb, + ult_ubound=None if ult_ub is None else ult_ub - 1, + ult_lbound=None if ult_lb is None else ult_lb - 1) # add SP1 spatially constant, but temporally correlated wel flux pars @@ -810,8 +821,6 @@ def mf6_freyberg_test(): print(pst.phi) #assert pst.phi < 1.0e-5, pst.phi - - # check mult files are in pst input files csv = os.path.join(template_ws, "mult2model_info.csv") df = pd.read_csv(csv, index_col=0) @@ -820,6 +829,15 @@ def mf6_freyberg_test(): set(df.loc[df.pp_file.notna()].mlt_file)) assert len(mults_not_linked_to_pst) == 0, print(mults_not_linked_to_pst) + # make sure the appropriate ult bounds have made it thru + df = pd.read_csv(os.path.join(template_ws,"mult2model_info.csv")) + print(df.columns) + df = df.loc[df.model_file.apply(lambda x: "npf_k_" in x),:] + print(df) + print(df.upper_bound) + print(df.lower_bound) + assert np.abs(float(df.upper_bound.min()) - 19.) < 1.0e-6,df.upper_bound.min() + assert np.abs(float(df.lower_bound.max()) - 3.) < 1.0e-6,df.lower_bound.max() def mf6_freyberg_shortnames_test(): import numpy as np @@ -1313,9 +1331,9 @@ def mf6_freyberg_direct_test(): raise Exception("recharge too diff") if __name__ == "__main__": - freyberg_test() + # freyberg_test() # freyberg_prior_build_test() - # mf6_freyberg_test() + mf6_freyberg_test() # mf6_freyberg_shortnames_test() # mf6_freyberg_da_test() # mf6_freyberg_direct_test() diff --git a/pyemu/utils/pst_from.py b/pyemu/utils/pst_from.py index a475b1a06..6655f813e 100644 --- a/pyemu/utils/pst_from.py +++ b/pyemu/utils/pst_from.py @@ -165,7 +165,7 @@ def parfile_relations(self): else pd.Series( {k: v for k, v in [['lbound', x.lower_bound]]}), axis=1) if lbound.nunique(0, False).gt(1).any(): - lb_max = lbound.min().fillna(self.ult_lbound_fill).to_dict() + lb_max = lbound.max().fillna(self.ult_lbound_fill).to_dict() pr.loc[g.index, 'lower_bound'] = g.use_cols.apply( lambda x: [lb_max['lbound{0}'.format(c)] for c in x] if x is not None else lb_max['lbound']) From a605e1d992108f5aebc3c3e36ec8d5806b7eb9e0 Mon Sep 17 00:00:00 2001 From: White Date: Thu, 20 Aug 2020 12:42:51 -0500 Subject: [PATCH 7/7] swapped a few 'is not' to '!=' --- pyemu/utils/helpers.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pyemu/utils/helpers.py b/pyemu/utils/helpers.py index cdf640ed8..0316bbb32 100644 --- a/pyemu/utils/helpers.py +++ b/pyemu/utils/helpers.py @@ -377,18 +377,18 @@ def calc_observation_ensemble_quantiles(ens, pst, quantiles, subset_obsnames=Non if subset_obsnames is not None: trimnames = subset_obsnames - if len(set(trimnames) - set(obs.index.values)) is not 0: + if len(set(trimnames) - set(obs.index.values)) != 0: raise Exception('the following names in subset_obsnames are not in the ensemble:\n' + ['{}\n'.format(i) for i in (set(trimnames) - set(obs.index.values))]) if subset_obsgroups is not None: - if len((set(subset_obsgroups) - set(pst.obs_groups))) is not 0: + if len((set(subset_obsgroups) - set(pst.obs_groups))) != 0: raise Exception('the following groups in subset_obsgroups are not in pst:\n' + ['{}\n'.format(i) for i in (set(subset_obsgroups) - set(pst.obs_groups))]) trimnames = obs.loc[obs.obgnme.isin(subset_obsgroups)].obsnme.tolist() - if len((set(trimnames) - set(obs.index.values)))is not 0: + if len((set(trimnames) - set(obs.index.values))) != 0: raise Exception('the following names in subset_obsnames are not in the ensemble:\n' + ['{}\n'.format(i) for i in (set(trimnames) - set(obs.index.values))]) # trim the data to subsets (or complete )