bugfix pst_utils.py: set write_input_files to use mp.pool for multipr…

autotest/pst_tests.py

@@ -470,6 +470,10 @@ def write_tables_test():
     group_names = {"w0": "wells t"}
     pst.write_par_summary_table(group_names=group_names)
     pst.write_obs_summary_table(group_names={"calhead": "calibration heads"})
+    pst.write_par_summary_table(filename='testpar.xlsx', group_names=group_names)
+    pst.write_par_summary_table(filename='testpar2.xlsx', group_names=group_names, report_in_linear_space=True)   
+    pst.write_obs_summary_table(filename = 'testobs.xlsx', group_names={"calhead": "calibration heads"})
+
 
 def test_e_clean():
     import os
@@ -926,7 +930,7 @@ def new_format_path_mechanics_test():
     # res_stats_test()
     # test_write_input_files()
     #add_obs_test()
-    add_pars_test()
+    #add_pars_test()
     # setattr_test()
 
     # add_pi_test()
@@ -951,3 +955,4 @@ def new_format_path_mechanics_test():
     # run_test()
     # rectify_pgroup_test()
     # sanity_check_test()
+    write_tables_test()

pyemu/pst/pst_handler.py

@@ -2934,17 +2934,20 @@ def plot(self, kind=None, **kwargs):
         """
         return plot_utils.pst_helper(self, kind, **kwargs)
 
-    def write_par_summary_table(self, filename=None, group_names=None, sigma_range=4.0):
-        """write a stand alone parameter summary latex table
+    def write_par_summary_table(self, filename=None, group_names=None, sigma_range=4.0, report_in_linear_space=False):
+        """write a stand alone parameter summary latex table or Excel sheet
 
 
         Args:
-            filename (`str`): latex filename. If None, use <case>.par.tex. If `filename` is "none", no table
-                is writtenDefault is None
+            filename (`str`): filename. If None, use <case>.par.tex to write as LaTeX. If filename extention is '.xls' or '.xlsx',
+                tries to write as an Excel file. If `filename` is "none", no table is written
+                Default is None
             group_names (`dict`): par group names : table names. For example {"w0":"well stress period 1"}.
                 Default is None
             sigma_range (`float`): number of standard deviations represented by parameter bounds.  Default
                 is 4.0, implying 95% confidence bounds
+            report_in_linear_space (`bool`): flag, if True, that reports all logtransformed values in linear
+                space. This renders standard deviation meaningless, so that column is skipped  
 
         Returns:
             `pandas.DataFrame`: the summary parameter group dataframe
@@ -2960,7 +2963,11 @@ def write_par_summary_table(self, filename=None, group_names=None, sigma_range=4
         par = self.parameter_data.copy()
         pargp = par.groupby(par.pargp).groups
         # cols = ["parval1","parubnd","parlbnd","stdev","partrans","pargp"]
-        cols = ["pargp", "partrans", "count", "parval1", "parubnd", "parlbnd", "stdev"]
+        if report_in_linear_space == True:
+            cols = ["pargp", "partrans", "count", "parval1", "parlbnd", "parubnd"]
+
+        else:
+            cols = ["pargp", "partrans", "count", "parval1", "parlbnd", "parubnd", "stdev"]
 
         labels = {
             "parval1": "initial value",
@@ -2973,10 +2980,14 @@ def write_par_summary_table(self, filename=None, group_names=None, sigma_range=4
         }
 
         li = par.partrans == "log"
-        par.loc[li, "parval1"] = par.parval1.loc[li].apply(np.log10)
-        par.loc[li, "parubnd"] = par.parubnd.loc[li].apply(np.log10)
-        par.loc[li, "parlbnd"] = par.parlbnd.loc[li].apply(np.log10)
-        par.loc[:, "stdev"] = (par.parubnd - par.parlbnd) / sigma_range
+        if True in li.values and report_in_linear_space == True:
+            print('Warning: because log-transformed values being reported in linear space, stdev NOT reported')
+
+        if report_in_linear_space == False:
+            par.loc[li, "parval1"] = par.parval1.loc[li].apply(np.log10)
+            par.loc[li, "parubnd"] = par.parubnd.loc[li].apply(np.log10)
+            par.loc[li, "parlbnd"] = par.parlbnd.loc[li].apply(np.log10)
+            par.loc[:, "stdev"] = (par.parubnd - par.parlbnd) / sigma_range
 
         data = {c: [] for c in cols}
         for pg, pnames in pargp.items():
@@ -3020,24 +3031,40 @@ def write_par_summary_table(self, filename=None, group_names=None, sigma_range=4
             return pargp_df
         if filename is None:
             filename = self.filename.replace(".pst", ".par.tex")
+        # if filename indicates an Excel format, try writing to Excel
+        if filename.lower().endswith('xlsx') or filename.lower().endswith('xls'):
+            try:
+                pargp_df.to_excel(filename, index=None)
+            except Exception as e:
+                if filename.lower().endswith('xlsx'):
+                    print('could not export {0} in Excel format. Try installing xlrd'.format(filename))
+                elif filename.lower().endswith('xls'):
+                    print('could not export {0} in Excel format. Try installing xlwt'.format(filename))
+                else:
+                    print('could not export {0} in Excel format.'.format(filename))
 
-        with open(filename, "w") as f:
-            f.write(preamble)
-            f.write("\\begin{center}\nParameter Summary\n\\end{center}\n")
-            f.write("\\begin{center}\n\\begin{landscape}\n")
-            pargp_df.to_latex(f, index=False, longtable=True)
-            f.write("\\end{landscape}\n")
-            f.write("\\end{center}\n")
-            f.write("\\end{document}\n")
+        else:
+            with open(filename, "w") as f:
+                f.write(preamble)
+                f.write("\\begin{center}\nParameter Summary\n\\end{center}\n")
+                f.write("\\begin{center}\n\\begin{landscape}\n")
+                pargp_df.to_latex(f, index=False, longtable=True)
+                f.write("\\end{landscape}\n")
+                f.write("\\end{center}\n")
+                f.write("\\end{document}\n")
         return pargp_df
 
     def write_obs_summary_table(self, filename=None, group_names=None):
-        """write a stand alone observation summary latex table
-
+        """write a stand alone observation summary latex table or Excel shet
+            filename (`str`): filename. If None, use <case>.par.tex to write as LaTeX. If filename extention is '.xls' or '.xlsx',
+                tries to write as an Excel file. If `filename` is "none", no table is written
+                Default is None
 
         Args:
-            filename (`str`): latex filename. If `filename` is "none", no table is written.
-                If None, use <case>.par.tex. Default is None
+            filename (`str`): filename. If `filename` is "none", no table is written.
+                If None, use <case>.obs.tex. If filename extention is '.xls' or '.xlsx',
+                tries to write as an Excel file.
+                Default is None
             group_names (`dict`): obs group names : table names. For example {"hds":"simulated groundwater level"}.
                 Default is None
 
@@ -3106,21 +3133,33 @@ def write_obs_summary_table(self, filename=None, group_names=None):
 
         if filename == "none":
             return obsg_df
-
         if filename is None:
             filename = self.filename.replace(".pst", ".obs.tex")
+        # if filename indicates an Excel format, try writing to Excel
+        if filename.lower().endswith('xlsx') or filename.lower().endswith('xls'):
+            try:
+                obsg_df.to_excel(filename, index=None)
+            except Exception as e:
+                if filename.lower().endswith('xlsx'):
+                    print('could not export {0} in Excel format. Try installing xlrd'.format(filename))
+                elif filename.lower().endswith('xls'):
+                    print('could not export {0} in Excel format. Try installing xlwt'.format(filename))
+                else:
+                    print('could not export {0} in Excel format.'.format(filename))
 
-        with open(filename, "w") as f:
 
-            f.write(preamble)
+        else:
+            with open(filename, "w") as f:
+
+                f.write(preamble)
 
-            f.write("\\begin{center}\nObservation Summary\n\\end{center}\n")
-            f.write("\\begin{center}\n\\begin{landscape}\n")
-            f.write("\\setlength{\\LTleft}{-4.0cm}\n")
-            obsg_df.to_latex(f, index=False, longtable=True)
-            f.write("\\end{landscape}\n")
-            f.write("\\end{center}\n")
-            f.write("\\end{document}\n")
+                f.write("\\begin{center}\nObservation Summary\n\\end{center}\n")
+                f.write("\\begin{center}\n\\begin{landscape}\n")
+                f.write("\\setlength{\\LTleft}{-4.0cm}\n")
+                obsg_df.to_latex(f, index=False, longtable=True)
+                f.write("\\end{landscape}\n")
+                f.write("\\end{center}\n")
+                f.write("\\end{document}\n")
 
         return obsg_df
 

pyemu/pst/pst_utils.py

@@ -453,19 +453,26 @@ def write_input_files(pst, pst_path="."):
     )  # the list of files broken down into chunks
     remainder = pairs[num_chunk_floor * chunk_len :].tolist()  # remaining files
     chunks = main_chunks + [remainder]
-    procs = []
-    for chunk in chunks:
-        # write_to_template(pst.parameter_data.parval1_trans,os.path.join(pst_path,tpl_file),
-        #                  os.path.join(pst_path,in_file))
-        p = mp.Process(
-            target=_write_chunk_to_template,
-            args=[chunk, pst.parameter_data.parval1_trans, pst_path],
-        )
-        p.start()
-        procs.append(p)
-    for p in procs:
-        p.join()
-
+#    procs = []
+#   for chunk in chunks:
+#        # write_to_template(pst.parameter_data.parval1_trans,os.path.join(pst_path,tpl_file),
+#        #                  os.path.join(pst_path,in_file))
+#        p = mp.Process(
+#            target=_write_chunk_to_template,
+#            args=[chunk, pst.parameter_data.parval1_trans, pst_path],
+#        )
+#        p.start()
+#        procs.append(p)
+#    for p in procs:
+#        p.join()
+    pool = mp.Pool()
+    x = [
+        pool.apply_async(_write_chunk_to_template, args=(chunk, pst.parameter_data.parval1_trans, pst_path))
+        for i, chunk in enumerate(chunks)
+    ]
+    [xx.get() for xx in x]
+    pool.close()
+    pool.join()
 
 def _write_chunk_to_template(chunk, parvals, pst_path):
     for tpl_file, in_file in chunk: