From a040fa1f4eb5e9a4dbb73b518d5f01f957da9ba4 Mon Sep 17 00:00:00 2001 From: Mike Fienen Date: Fri, 20 Nov 2020 12:29:20 -0600 Subject: [PATCH 1/3] bugfix pst_utils.py: set write_input_files to use mp.pool for multiprocessing thus to convey error messages from subprocesses back to main process --- pyemu/pst/pst_utils.py | 33 ++++++++++++++++++++------------- 1 file changed, 20 insertions(+), 13 deletions(-) diff --git a/pyemu/pst/pst_utils.py b/pyemu/pst/pst_utils.py index b52efb363..7ea231ecc 100644 --- a/pyemu/pst/pst_utils.py +++ b/pyemu/pst/pst_utils.py @@ -453,19 +453,26 @@ def write_input_files(pst, pst_path="."): ) # the list of files broken down into chunks remainder = pairs[num_chunk_floor * chunk_len :].tolist() # remaining files chunks = main_chunks + [remainder] - procs = [] - for chunk in chunks: - # write_to_template(pst.parameter_data.parval1_trans,os.path.join(pst_path,tpl_file), - # os.path.join(pst_path,in_file)) - p = mp.Process( - target=_write_chunk_to_template, - args=[chunk, pst.parameter_data.parval1_trans, pst_path], - ) - p.start() - procs.append(p) - for p in procs: - p.join() - +# procs = [] +# for chunk in chunks: +# # write_to_template(pst.parameter_data.parval1_trans,os.path.join(pst_path,tpl_file), +# # os.path.join(pst_path,in_file)) +# p = mp.Process( +# target=_write_chunk_to_template, +# args=[chunk, pst.parameter_data.parval1_trans, pst_path], +# ) +# p.start() +# procs.append(p) +# for p in procs: +# p.join() + pool = mp.Pool() + x = [ + pool.apply_async(_write_chunk_to_template, args=(chunk, pst.parameter_data.parval1_trans, pst_path)) + for i, chunk in enumerate(chunks) + ] + [xx.get() for xx in x] + pool.close() + pool.join() def _write_chunk_to_template(chunk, parvals, pst_path): for tpl_file, in_file in chunk: From a297b8f0588023b73acb1ad2bf1bd09b0adaebfa Mon Sep 17 00:00:00 2001 From: Mike Fienen Date: Mon, 23 Nov 2020 11:19:23 -0600 Subject: [PATCH 2/3] feat(pst_handler.py): added support to export obs/par summary tables in Excel format. Also, added a flag to write_par_summary_table() (report_in_linear_space) which, if True, reports log-transformed values and bounds in linear space and does not report standard deviation since that quantity is only relevant in transformed space. --- pyemu/pst/pst_handler.py | 101 +++++++++++++++++++++++++++------------ 1 file changed, 70 insertions(+), 31 deletions(-) diff --git a/pyemu/pst/pst_handler.py b/pyemu/pst/pst_handler.py index be8d03ace..dd3b55580 100644 --- a/pyemu/pst/pst_handler.py +++ b/pyemu/pst/pst_handler.py @@ -2934,17 +2934,20 @@ def plot(self, kind=None, **kwargs): """ return plot_utils.pst_helper(self, kind, **kwargs) - def write_par_summary_table(self, filename=None, group_names=None, sigma_range=4.0): - """write a stand alone parameter summary latex table + def write_par_summary_table(self, filename=None, group_names=None, sigma_range=4.0, report_in_linear_space=False): + """write a stand alone parameter summary latex table or Excel sheet Args: - filename (`str`): latex filename. If None, use .par.tex. If `filename` is "none", no table - is writtenDefault is None + filename (`str`): filename. If None, use .par.tex to write as LaTeX. If filename extention is '.xls' or '.xlsx', + tries to write as an Excel file. If `filename` is "none", no table is written + Default is None group_names (`dict`): par group names : table names. For example {"w0":"well stress period 1"}. Default is None sigma_range (`float`): number of standard deviations represented by parameter bounds. Default is 4.0, implying 95% confidence bounds + report_in_linear_space (`bool`): flag, if True, that reports all logtransformed values in linear + space. This renders standard deviation meaningless, so that column is skipped Returns: `pandas.DataFrame`: the summary parameter group dataframe @@ -2960,7 +2963,11 @@ def write_par_summary_table(self, filename=None, group_names=None, sigma_range=4 par = self.parameter_data.copy() pargp = par.groupby(par.pargp).groups # cols = ["parval1","parubnd","parlbnd","stdev","partrans","pargp"] - cols = ["pargp", "partrans", "count", "parval1", "parubnd", "parlbnd", "stdev"] + if report_in_linear_space == True: + cols = ["pargp", "partrans", "count", "parval1", "parlbnd", "parubnd"] + + else: + cols = ["pargp", "partrans", "count", "parval1", "parlbnd", "parlbnd", "stdev"] labels = { "parval1": "initial value", @@ -2973,10 +2980,14 @@ def write_par_summary_table(self, filename=None, group_names=None, sigma_range=4 } li = par.partrans == "log" - par.loc[li, "parval1"] = par.parval1.loc[li].apply(np.log10) - par.loc[li, "parubnd"] = par.parubnd.loc[li].apply(np.log10) - par.loc[li, "parlbnd"] = par.parlbnd.loc[li].apply(np.log10) - par.loc[:, "stdev"] = (par.parubnd - par.parlbnd) / sigma_range + if True in li.values: + print('Warning: because log-transformed values being reported in linear space, stdev NOT reported') + + if report_in_linear_space == False: + par.loc[li, "parval1"] = par.parval1.loc[li].apply(np.log10) + par.loc[li, "parubnd"] = par.parubnd.loc[li].apply(np.log10) + par.loc[li, "parlbnd"] = par.parlbnd.loc[li].apply(np.log10) + par.loc[:, "stdev"] = (par.parubnd - par.parlbnd) / sigma_range data = {c: [] for c in cols} for pg, pnames in pargp.items(): @@ -3020,24 +3031,40 @@ def write_par_summary_table(self, filename=None, group_names=None, sigma_range=4 return pargp_df if filename is None: filename = self.filename.replace(".pst", ".par.tex") + # if filename indicates an Excel format, try writing to Excel + if filename.lower().endswith('xlsx') or filename.lower().endswith('xls'): + try: + pargp_df.to_excel(filename, index=None) + except Exception as e: + if filename.lower().endswith('xlsx'): + print('could not export {0} in Excel format. Try installing xlrd'.format(filename)) + elif filename.lower().endswith('xls'): + print('could not export {0} in Excel format. Try installing xlwt'.format(filename)) + else: + print('could not export {0} in Excel format.'.format(filename)) - with open(filename, "w") as f: - f.write(preamble) - f.write("\\begin{center}\nParameter Summary\n\\end{center}\n") - f.write("\\begin{center}\n\\begin{landscape}\n") - pargp_df.to_latex(f, index=False, longtable=True) - f.write("\\end{landscape}\n") - f.write("\\end{center}\n") - f.write("\\end{document}\n") + else: + with open(filename, "w") as f: + f.write(preamble) + f.write("\\begin{center}\nParameter Summary\n\\end{center}\n") + f.write("\\begin{center}\n\\begin{landscape}\n") + pargp_df.to_latex(f, index=False, longtable=True) + f.write("\\end{landscape}\n") + f.write("\\end{center}\n") + f.write("\\end{document}\n") return pargp_df def write_obs_summary_table(self, filename=None, group_names=None): - """write a stand alone observation summary latex table - + """write a stand alone observation summary latex table or Excel shet + filename (`str`): filename. If None, use .par.tex to write as LaTeX. If filename extention is '.xls' or '.xlsx', + tries to write as an Excel file. If `filename` is "none", no table is written + Default is None Args: - filename (`str`): latex filename. If `filename` is "none", no table is written. - If None, use .par.tex. Default is None + filename (`str`): filename. If `filename` is "none", no table is written. + If None, use .obs.tex. If filename extention is '.xls' or '.xlsx', + tries to write as an Excel file. + Default is None group_names (`dict`): obs group names : table names. For example {"hds":"simulated groundwater level"}. Default is None @@ -3106,21 +3133,33 @@ def write_obs_summary_table(self, filename=None, group_names=None): if filename == "none": return obsg_df - if filename is None: filename = self.filename.replace(".pst", ".obs.tex") + # if filename indicates an Excel format, try writing to Excel + if filename.lower().endswith('xlsx') or filename.lower().endswith('xls'): + try: + pargp_df.to_excel(filename, index=None) + except Exception as e: + if filename.lower().endswith('xlsx'): + print('could not export {0} in Excel format. Try installing xlrd'.format(filename)) + elif filename.lower().endswith('xls'): + print('could not export {0} in Excel format. Try installing xlwt'.format(filename)) + else: + print('could not export {0} in Excel format.'.format(filename)) - with open(filename, "w") as f: - f.write(preamble) + else: + with open(filename, "w") as f: + + f.write(preamble) - f.write("\\begin{center}\nObservation Summary\n\\end{center}\n") - f.write("\\begin{center}\n\\begin{landscape}\n") - f.write("\\setlength{\\LTleft}{-4.0cm}\n") - obsg_df.to_latex(f, index=False, longtable=True) - f.write("\\end{landscape}\n") - f.write("\\end{center}\n") - f.write("\\end{document}\n") + f.write("\\begin{center}\nObservation Summary\n\\end{center}\n") + f.write("\\begin{center}\n\\begin{landscape}\n") + f.write("\\setlength{\\LTleft}{-4.0cm}\n") + obsg_df.to_latex(f, index=False, longtable=True) + f.write("\\end{landscape}\n") + f.write("\\end{center}\n") + f.write("\\end{document}\n") return obsg_df From f36e3ed6c6014229fb7650fdae291d1edda23d5c Mon Sep 17 00:00:00 2001 From: Mike Fienen Date: Mon, 23 Nov 2020 15:32:44 -0600 Subject: [PATCH 3/3] bugfix (pst_handler.py): fixed introduced bug. Also added tests for the new write_summary_table options for both pars and obs --- autotest/pst_tests.py | 7 ++++++- pyemu/pst/pst_handler.py | 6 +++--- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/autotest/pst_tests.py b/autotest/pst_tests.py index c2a620890..19af476af 100644 --- a/autotest/pst_tests.py +++ b/autotest/pst_tests.py @@ -470,6 +470,10 @@ def write_tables_test(): group_names = {"w0": "wells t"} pst.write_par_summary_table(group_names=group_names) pst.write_obs_summary_table(group_names={"calhead": "calibration heads"}) + pst.write_par_summary_table(filename='testpar.xlsx', group_names=group_names) + pst.write_par_summary_table(filename='testpar2.xlsx', group_names=group_names, report_in_linear_space=True) + pst.write_obs_summary_table(filename = 'testobs.xlsx', group_names={"calhead": "calibration heads"}) + def test_e_clean(): import os @@ -926,7 +930,7 @@ def new_format_path_mechanics_test(): # res_stats_test() # test_write_input_files() #add_obs_test() - add_pars_test() + #add_pars_test() # setattr_test() # add_pi_test() @@ -951,3 +955,4 @@ def new_format_path_mechanics_test(): # run_test() # rectify_pgroup_test() # sanity_check_test() + write_tables_test() \ No newline at end of file diff --git a/pyemu/pst/pst_handler.py b/pyemu/pst/pst_handler.py index dd3b55580..3d6fce27b 100644 --- a/pyemu/pst/pst_handler.py +++ b/pyemu/pst/pst_handler.py @@ -2967,7 +2967,7 @@ def write_par_summary_table(self, filename=None, group_names=None, sigma_range=4 cols = ["pargp", "partrans", "count", "parval1", "parlbnd", "parubnd"] else: - cols = ["pargp", "partrans", "count", "parval1", "parlbnd", "parlbnd", "stdev"] + cols = ["pargp", "partrans", "count", "parval1", "parlbnd", "parubnd", "stdev"] labels = { "parval1": "initial value", @@ -2980,7 +2980,7 @@ def write_par_summary_table(self, filename=None, group_names=None, sigma_range=4 } li = par.partrans == "log" - if True in li.values: + if True in li.values and report_in_linear_space == True: print('Warning: because log-transformed values being reported in linear space, stdev NOT reported') if report_in_linear_space == False: @@ -3138,7 +3138,7 @@ def write_obs_summary_table(self, filename=None, group_names=None): # if filename indicates an Excel format, try writing to Excel if filename.lower().endswith('xlsx') or filename.lower().endswith('xls'): try: - pargp_df.to_excel(filename, index=None) + obsg_df.to_excel(filename, index=None) except Exception as e: if filename.lower().endswith('xlsx'): print('could not export {0} in Excel format. Try installing xlrd'.format(filename))