Hotfix noheader list

.github/workflows/ci.yml

@@ -114,7 +114,7 @@ jobs:
             exit 1
         fi
         cp -r bin/$d/. "$HOME/.local/bin/"
-        git clone -b develop --depth 1 https://github.com/usgs/pestpp
+        git clone -b feat_newtools --depth 1 https://github.com/usgs/pestpp
         cd pestpp
         mkdir build && cd build
         if [[ "$RUNNER_OS" == "Windows" ]]; then

autotest/pst_from_tests.py

@@ -39,7 +39,6 @@
 mf_exe_name = os.path.basename(mf_exe_path)
 mf6_exe_name = os.path.basename(mf6_exe_path)
 
-
 def freyberg_test():
     import numpy as np
     import pandas as pd
@@ -305,7 +304,8 @@ def freyberg_prior_build_test():
                  longnames=True, spatial_reference=sr,
                  zero_based=False)
     pf.extra_py_imports.append('flopy')
-    pf.mod_sys_cmds.append("which python")
+    if "linux" in platform.platform().lower():
+        pf.mod_sys_cmds.append("which python")
     # obs
     #   using tabular style model output
     #   (generated by pyemu.gw_utils.setup_hds_obs())
@@ -2528,7 +2528,7 @@ def pstfrom_profile():
     pr.print_stats(sort="cumtime")
 
 
-def mf6_freyberg_arr_obs_test():
+def mf6_freyberg_arr_obs_and_headerless_test():
     import numpy as np
     import pandas as pd
     pd.set_option('display.max_rows', 500)
@@ -2572,6 +2572,14 @@ def mf6_freyberg_arr_obs_test():
                  longnames=True, spatial_reference=sr,
                  zero_based=False, start_datetime="1-1-2018")
 
+    list_file = "freyberg6.wel_stress_period_data_1.txt"
+    df = pd.read_csv(os.path.join(template_ws,list_file),header=None,delim_whitespace=True)
+    df.loc[:,4] = 4
+    df.loc[:,5] = 5
+    df.to_csv(os.path.join(template_ws,list_file),sep=" ",index=False,header=False)
+    pf.add_observations(list_file, index_cols=[0, 1, 2], use_cols=[3,5], ofile_skip=0, includes_header=False,
+                        prefix="welobs")
+
     v = pyemu.geostats.ExpVario(contribution=1.0, a=1000)
     gr_gs = pyemu.geostats.GeoStruct(variograms=v)
     rch_temporal_gs = pyemu.geostats.GeoStruct(variograms=pyemu.geostats.ExpVario(contribution=1.0, a=60))
@@ -2599,11 +2607,15 @@ def mf6_freyberg_arr_obs_test():
             pf.add_observations(arr_file,zone_array=ib)
             arr_dict[arr_file] = np.loadtxt(pf.new_d / arr_file)
 
+
+
     # add model run command
     pf.mod_sys_cmds.append("mf6")
     print(pf.mult_files)
     print(pf.org_files)
 
+
+
     # build pest
     pst = pf.build_pst('freyberg.pst')
     pe = pf.draw(100,use_specsim=True)
@@ -2631,22 +2643,39 @@ def mf6_freyberg_arr_obs_test():
         print(fname,pval,aval)
         assert pval == aval,"{0},{1},{2}".format(fname,pval,aval)
 
+    df = pd.read_csv(os.path.join(template_ws,list_file),header=None,delim_whitespace=True)
+    print(df)
+    wobs = obs.loc[obs.obsnme.str.contains("welobs"),:]
+    print(wobs)
+    fvals = df.iloc[:,3]
+    pvals = wobs.loc[:,"obsval"].iloc[:df.shape[0]]
+    d = fvals.values - pvals.values
+    print(d)
+    assert d.sum() == 0
+    fvals = df.iloc[:, 5]
+    pvals = wobs.loc[:, "obsval"].iloc[df.shape[0]:]
+    d = fvals.values - pvals.values
+    print(d)
+    assert d.sum() == 0
+
+
 
 if __name__ == "__main__":
     #invest()
-    # freyberg_test()
+    freyberg_test()
     #freyberg_prior_build_test()
-    mf6_freyberg_test()
+    #mf6_freyberg_test()
     #mf6_freyberg_shortnames_test()
     #mf6_freyberg_direct_test()
     #mf6_freyberg_varying_idomain()
     #xsec_test()
     #mf6_freyberg_short_direct_test()
-    # tpf = TestPstFrom()
-    # tpf.setup()
-    # tpf.test_add_direct_array_parameters()
+    tpf = TestPstFrom()
+    tpf.setup()
+    tpf.test_add_direct_array_parameters()
+    tpf.add
     #pstfrom_profile()
-    #mf6_freyberg_arr_obs_test()
+    #mf6_freyberg_arr_obs_and_headerless_test()\
 
 
 

autotest/pst_tests.py

@@ -400,7 +400,7 @@ def add_obs_test():
         f.write("l1 w  !{0}!\n".format("crap1"))
     with open(out_file, "w") as f:
         f.write("junk1  {0:8.2f} \n".format(oval))
-    pst.add_observations(ins_file, out_file, pst_path="temp")
+    pst.add_observations(ins_file, out_file)
     assert nobs + 1 == pst.nobs
     assert "crap1" in pst.observation_data.obsnme
     assert os.path.join("temp", "crap.out") in pst.output_files, str(pst.output_files)
@@ -968,7 +968,7 @@ def read_in_tpl_test():
     # write_tables_test()
     # res_stats_test()
     # test_write_input_files()
-    #add_obs_test()
+    add_obs_test()
     #add_pars_test()
     # setattr_test()
 
@@ -1002,5 +1002,6 @@ def read_in_tpl_test():
     #tpl_ins_test()
     #process_output_files_test()
     #comments_test()
-    read_in_tpl_test()
-    comments_test()
+    #read_in_tpl_test()
+    #comments_test()
+    #csv_to_ins_test()

pyemu/pst/pst_utils.py

@@ -1070,44 +1070,52 @@ def csv_to_ins_file(
         if isinstance(only_rows, str):  # incase it is a single name
             only_rows = [only_rows]
         only_rows = set(only_rows)
-    only_cols = {c.lower() if isinstance(c, str) else c for c in only_cols}
+    only_rows = {r.lower() if isinstance(r, str) else r for r in only_rows}
 
     # process the row labels, handling duplicates
     rlabels = []
     row_visit = {}
     only_rlabels = []
-    for rname in df.index:
-        rname = str(rname).strip().lower()
-
+    for rname_org in df.index:
+        rname = str(rname_org).strip().lower()
         if rname in row_visit:
-            rsuffix = str(int(row_visit[rname] + 1))
+            if longnames:
+                rsuffix = "_"+str(int(row_visit[rname] + 1))
+            else:
+                rsuffix = str(int(row_visit[rname] + 1))
             row_visit[rname] += 1
         else:
             row_visit[rname] = 1
             rsuffix = ""
         rlabel = rname + rsuffix
         rlabels.append(rlabel)
-        if rname in only_rows:
+        if rname in only_rows or rname_org in only_rows:
             only_rlabels.append(rlabel)
     only_rlabels = set(only_rlabels)
 
     # process the col labels, handling duplicates
     clabels = []
     col_visit = {}
     only_clabels = []
-    for cname in df.columns:
-        cname = str(cname).strip().lower()
+    for cname_org in df.columns:
+        cname = str(cname_org).strip().lower()
         if cname in col_visit:
-            csuffix = str(int(col_visit[cname] + 1))
+            if longnames:
+                csuffix = "_"+str(int(col_visit[cname] + 1))
+            else:
+                csuffix = str(int(col_visit[cname] + 1))
             col_visit[cname] += 1
         else:
             col_visit[cname] = 1
             csuffix = ""
         clabel = cname + csuffix
         clabels.append(clabel)
-        if cname in only_cols:
+        if cname in only_cols or cname_org in only_cols:
             only_clabels.append(clabel)
     only_clabels = set(only_clabels)
+    if len(only_clabels) == 0:
+        print("only_cols:",only_cols)
+        raise Exception("csv_to_ins_file(): only_clabels is empty")
 
     if ins_filename is None:
         if not isinstance(csv_filename, str):
@@ -1128,11 +1136,25 @@ def csv_to_ins_file(
             f.write("l1\n")  # skip the row (index) label
         for i, rlabel in enumerate(rlabels):  # loop over rows
             f.write("l1")
+
             c_count = 0
+            line = ''
             for j, clabel in enumerate(clabels):  # loop over columns
-                oname = ""
-                if c_count < only_clabels_len:  # if we haven't yet set up all obs
-                    if rlabel in only_rlabels and clabel in only_clabels:
+
+                if j == 0:
+                    # if first col and input file has an index need additional spacer
+                    if includes_index:
+                        if sep == ",":
+                            # f.write(f" {marker},{marker}")
+                            line += f" {marker},{marker}"
+                        else:
+                            # f.write(" !dum!")
+                            line += " !dum! "
+
+
+                if c_count < only_clabels_len:
+                    if clabel in only_clabels and rlabel in only_rlabels:
+                        oname = ""
                         # define obs names
                         if not prefix_is_str:
                             nprefix = prefix[c_count]
@@ -1171,22 +1193,19 @@ def csv_to_ins_file(
                         ognames.append(ngpname)  # add to list of group names
                         # start defining string to write in ins
                         oname = f" !{oname}!"
+                        line += f" {oname} "
+                        if j < len(clabels) - 1:
+                            if sep == ",":
+                                line += f" {marker},{marker} "
+                            #else:
+                            #    line += " !dum! "
                         c_count += 1
-                    # else:  # not a requested observation; add spacer
-                    if j < clabels_len - 1:
+                    elif j < len(clabels) - 1: # this isnt a row-col to observationalize (nice word!)
                         if sep == ",":
-                            oname = f"{oname} {marker},{marker}"
+                            line += f" {marker},{marker} "
                         else:
-                            oname = f"{oname} w"
-                    if j == 0:
-                        # if first col and input file has an index need additional spacer
-                        if includes_index:
-                            if sep == ",":
-                                f.write(f" {marker},{marker}")
-                            else:
-                                f.write(" w")
-                    f.write(oname)
-            f.write("\n")
+                            line += " !dum! "
+            f.write(line+"\n")
     odf = pd.DataFrame(
         {"obsnme": onames, "obsval": ovals, "obgnme": ognames}, index=onames
     ).dropna(
@@ -1479,11 +1498,12 @@ def _execute_ins_line(self, ins_line, ins_lcount):
                 try:
                     val = float(val_str)
                 except Exception as e:
-                    self.throw_out_error(
-                        "casting string '{0}' to float for instruction '{1}'".format(
-                            val_str, ins
+                    if oname != "dum":
+                        self.throw_out_error(
+                            "casting string '{0}' to float for instruction '{1}'".format(
+                                val_str, ins
+                            )
                         )
-                    )
 
                 if oname != "dum":
                     val_dict[oname] = val
@@ -1566,11 +1586,12 @@ def _execute_ins_line(self, ins_line, ins_lcount):
                 try:
                     val = float(val_str)
                 except Exception as e:
-                    self.throw_out_error(
-                        "casting string '{0}' to float for instruction '{1}'".format(
-                            val_str, ins
+                    if oname != "dum":
+                        self.throw_out_error(
+                            "casting string '{0}' to float for instruction '{1}'".format(
+                                val_str, ins
+                            )
                         )
-                    )
 
                 if oname != "dum":
                     val_dict[oname] = val
@@ -1624,11 +1645,12 @@ def _execute_ins_line(self, ins_line, ins_lcount):
                 try:
                     val = float(val_str)
                 except Exception as e:
-                    self.throw_out_error(
-                        "casting string '{0}' to float for instruction '{1}'".format(
-                            val_str, ins
+                    if oname != "dum":
+                        self.throw_out_error(
+                            "casting string '{0}' to float for instruction '{1}'".format(
+                                val_str, ins
+                            )
                         )
-                    )
 
                 if oname != "dum":
                     val_dict[oname] = val

pyemu/utils/pst_from.py

@@ -1109,6 +1109,7 @@ def add_observations(
         rebuild_pst=False,
         obsgp=True,
         zone_array=None,
+        includes_header=True,
     ):
         """
         Add values in output files as observations to PstFrom object
@@ -1127,6 +1128,8 @@ def add_observations(
                 new obs
             zone_array (`np.ndarray`): array defining spatial limits or zones
                 for array-style observations. Default is None
+            includes_header (`bool`): flag indicating that the list file includes a
+                header row.  Default is True.
 
         Returns:
             `Pandas.DataFrame`: dataframe with info for new observations
@@ -1249,7 +1252,7 @@ def add_observations(
                 only_cols=use_cols,
                 only_rows=use_rows,
                 marker="~",
-                includes_header=True,
+                includes_header=includes_header,
                 includes_index=False,
                 prefix=prefix,
                 longnames=self.longnames,