Hotfix obs shortnames

autotest/pst_from_tests.py

@@ -887,12 +887,106 @@ def mf6_freyberg_shortnames_test():
                  remove_existing=True,
                  longnames=False, spatial_reference=sr,
                  zero_based=False,start_datetime="1-1-2018")
+
+    df = pd.read_csv(os.path.join(tmp_model_ws, "heads.csv"), index_col=0)
+    try:
+        pf.add_observations("heads.csv", insfile="heads.csv.ins", index_cols="time", use_cols=list(df.columns.values),
+                            prefix="hds_too_long_tho")
+    except:
+        pass
+    else:
+        raise Exception("should have failed")
+
+    pf = PstFrom(original_d=tmp_model_ws, new_d=template_ws,
+                 remove_existing=True,
+                 longnames=False, spatial_reference=sr,
+                 zero_based=False, start_datetime="1-1-2018")
+    try:
+        pf.add_parameters(filenames="freyberg6.sfr_packagedata.txt", par_name_base="rhk_still_too_long",
+                          pargp="sfr_rhk", index_cols=[0, 1, 2, 3], use_cols=[9], upper_bound=10., lower_bound=0.1,
+                          par_type="grid")
+    except:
+        pass
+    else:
+        raise Exception("should have failed")
+
+    pf = PstFrom(original_d=tmp_model_ws, new_d=template_ws,
+                 remove_existing=True,
+                 longnames=False, spatial_reference=sr,
+                 zero_based=False, start_datetime="1-1-2018")
+    try:
+        pf.add_parameters(filenames="freyberg6.sfr_packagedata.txt", par_name_base="rhk_still_too_long",
+                          pargp="sfr_rhk", index_cols=[0, 1, 2, 3], use_cols=[9], upper_bound=10., lower_bound=0.1,
+                          par_type="grid", par_style="direct")
+    except:
+        pass
+    else:
+        raise Exception("should have failed")
+
+    pf = PstFrom(original_d=tmp_model_ws, new_d=template_ws,
+                 remove_existing=True,
+                 longnames=False, spatial_reference=sr,
+                 zero_based=False, start_datetime="1-1-2018")
+    try:
+        pf.add_parameters(filenames="freyberg6.ic_strt_layer1.txt", par_name_base="rhk_still_too_long",
+                          pargp="sfr_rhk", upper_bound=10., lower_bound=0.1,
+                          par_type="pilotpoints")
+    except:
+         pass
+    else:
+         raise Exception("should have failed")
+
+    pf = PstFrom(original_d=tmp_model_ws, new_d=template_ws,
+                 remove_existing=True,
+                 longnames=False, spatial_reference=sr,
+                 zero_based=False, start_datetime="1-1-2018")
+    try:
+        pf.add_parameters(filenames="freyberg6.ic_strt_layer1.txt", par_name_base="rhk_still_too_long",
+                          pargp="sfr_rhk", upper_bound=10., lower_bound=0.1,
+                          par_type="grid")
+    except:
+        pass
+    else:
+        raise Exception("should have failed")
+
+    try:
+        pf.add_parameters(filenames="freyberg6.ic_strt_layer1.txt", par_name_base="rhk_still_too_long",
+                          pargp="sfr_rhk", upper_bound=10., lower_bound=0.1,
+                          par_type="grid",par_style="direct")
+    except:
+        pass
+    else:
+        raise Exception("should have failed")
+
+
+    try:
+        pf.add_parameters(filenames="freyberg6.ic_strt_layer1.txt", par_name_base="rhk_still_too_long",
+                          pargp="sfr_rhk", upper_bound=10., lower_bound=0.1,
+                          par_type="constant")
+    except:
+        pass
+    else:
+        raise Exception("should have failed")
+
+    try:
+        pf.add_parameters(filenames="freyberg6.ic_strt_layer1.txt", par_name_base="rhk_still_too_long",
+                          pargp="sfr_rhk", upper_bound=10., lower_bound=0.1,
+                          par_type="zone")
+    except:
+        pass
+    else:
+        raise Exception("should have failed")
+
     # obs
     #   using tabular style model output
     #   (generated by pyemu.gw_utils.setup_hds_obs())
     # pf.add_observations('freyberg.hds.dat', insfile='freyberg.hds.dat.ins2',
     #                     index_cols='obsnme', use_cols='obsval', prefix='hds')
 
+    pf = PstFrom(original_d=tmp_model_ws, new_d=template_ws,
+                 remove_existing=True,
+                 longnames=False, spatial_reference=sr,
+                 zero_based=False, start_datetime="1-1-2018")
     df = pd.read_csv(os.path.join(tmp_model_ws,"heads.csv"),index_col=0)
     pf.add_observations("heads.csv",insfile="heads.csv.ins",index_cols="time",use_cols=list(df.columns.values),prefix="hds")
     df = pd.read_csv(os.path.join(tmp_model_ws, "sfr.csv"), index_col=0)
@@ -947,6 +1041,9 @@ def mf6_freyberg_shortnames_test():
     # build pest
     pst = pf.build_pst('freyberg.pst')
 
+    assert pst.parameter_data.parnme.apply(lambda x: len(x)).max() <= 12
+    assert pst.observation_data.obsnme.apply(lambda x: len(x)).max() <= 20
+
     num_reals = 100
     pe = pf.draw(num_reals, use_specsim=True)
     pe.to_binary(os.path.join(template_ws, "prior.jcb"))
@@ -2294,5 +2391,5 @@ def test_get_filepath():
     # mf6_freyberg_da_test()
     #mf6_freyberg_direct_test()
     #mf6_freyberg_varying_idomain()
-    xsec_test()
-
+    #xsec_test()
+    mf6_freyberg_short_direct_test()

autotest/pst_tests.py

@@ -917,6 +917,15 @@ def new_format_path_mechanics_test():
     assert options["sep"] == "w", options
 
 
+def pi_helper_test():
+    import os
+    import numpy as np
+    import pyemu
+
+    pst = pyemu.Pst(os.path.join("pst","zoned_nz_64.pst"))
+
+
+
 if __name__ == "__main__":
 
     # process_output_files_test()
@@ -955,4 +964,5 @@ def new_format_path_mechanics_test():
     # run_test()
     # rectify_pgroup_test()
     # sanity_check_test()
-    write_tables_test()
+    #write_tables_test()
+    pi_helper_test()

pyemu/pst/pst_utils.py

@@ -1031,8 +1031,13 @@ def csv_to_ins_file(
                             nname = "{0}_usecol:{1}".format(nprefix, clabel)
                             oname = "{0}_{1}".format(nname, rlabel)
                         else:
-                            nname = nprefix + clabel
-                            oname = nprefix + rlabel + "_" + clabel
+                            nname = nprefix + clabel.replace(" ","").replace("_","")
+                            oname = nprefix + rlabel.replace(" ","").replace("_","") +\
+                                    clabel.replace(" ","").replace("_","")
+                            if len(oname) > 20:
+                                raise Exception("csv_to_ins_file(): cant form observation name " + \
+                                                " for prefix '{0}' , row '{1}', col '{2}' in less than 20 chars".
+                                                format(nprefix,rlabel,clabel))
                         onames.append(oname)  # append list of obs
                         ovals.append(vals[i, j])  # store current obs val
                         # defin group name

pyemu/utils/pp_utils.py

@@ -490,7 +490,7 @@ def pilot_points_to_tpl(pp_file, tpl_file=None, name_prefix=None, longnames=Fals
                 too_long.append(name)
         if len(too_long) > 0:
             raise Exception(
-                "the following parameter names are too long:" ",".join(too_long)
+                "the following parameter names are too long:" + ",".join(too_long)
             )
         tpl_entries = ["~    {0}    ~".format(name) for name in names]
         pp_df.loc[:, "tpl"] = tpl_entries

pyemu/utils/pst_from.py

@@ -1056,7 +1056,7 @@ def add_observations(
                 includes_header=True,
                 includes_index=False,
                 prefix=prefix,
-                longnames=True,
+                longnames=self.longnames,
                 head_lines_len=lenhead,
                 sep=sep,
                 gpname=obsgp,
@@ -2347,12 +2347,12 @@ def _write_direct_df_tpl(
     # TODO much of this duplicates what is in _get_tpl_or_ins_df() -- could posssibly be consolidated
     # work out the union of indices across all dfs
 
-    sidx = set()
+    sidx = []
 
-    didx = set(df.loc[:, index_cols].apply(lambda x: tuple(x), axis=1))
-    sidx.update(didx)
+    didx = df.loc[:, index_cols].apply(lambda x: tuple(x), axis=1)
+    sidx.extend(didx)
 
-    df_ti = pd.DataFrame({"sidx": list(sidx)}, columns=["sidx"])
+    df_ti = pd.DataFrame({"sidx": sidx}, columns=["sidx"])
     # get some index strings for naming
     if longnames:
         j = "_"
@@ -2429,6 +2429,7 @@ def _write_direct_df_tpl(
                 if suffix != "":
                     df_ti.loc[:, use_col] += suffix
 
+
             _check_diff(df.loc[:, use_col].values, in_filename)
             df_ti.loc[:, "parval1_{0}".format(use_col)] = df.loc[:, use_col][0]
 
@@ -2484,6 +2485,13 @@ def _write_direct_df_tpl(
                 "should be 'constant','zone', "
                 "or 'grid', not '{0}'".format(typ)
             )
+
+        if not longnames:
+            if df_ti.loc[:,use_col].apply(lambda x: len(x)).max() > 12:
+                too_long = df_ti.loc[:,use_col].apply(lambda x: len(x)) > 12
+                print(too_long)
+                self.logger.lraise("_write_direct_df_tpl(): couldnt form short par names")
+
         direct_tpl_df.loc[:, use_col] = (
             df_ti.loc[:, use_col].apply(lambda x: "~ {0} ~".format(x)).values
         )
@@ -2693,6 +2701,12 @@ def _get_tpl_or_ins_df(
                 "should be 'constant','zone', "
                 "or 'grid', not '{0}'".format(typ)
             )
+
+        if not longnames:
+            if df_ti.loc[:,use_col].apply(lambda x: len(x)).max() > 12:
+                too_long = df_ti.loc[:,use_col].apply(lambda x: len(x)) > 12
+                print(too_long)
+                self.logger.lraise("_get_tpl_or_ins_df(): couldnt form short par names")
     return df_ti