From 89b5b6af5fac9811a9334e202c810b1f8a6fdda8 Mon Sep 17 00:00:00 2001 From: Herb Schilling Date: Mon, 6 Nov 2023 15:45:54 -0500 Subject: [PATCH 01/22] Have code and tests working for having default locations of optimizer specific files be put into reports directory --- openmdao/core/constants.py | 7 +++ openmdao/core/problem.py | 6 +++ openmdao/drivers/pyoptsparse_driver.py | 50 ++++++++++++++++- .../drivers/tests/test_pyoptsparse_driver.py | 54 +++++++++++++++++++ 4 files changed, 115 insertions(+), 2 deletions(-) diff --git a/openmdao/core/constants.py b/openmdao/core/constants.py index 142b34c828..ed1ff954c1 100644 --- a/openmdao/core/constants.py +++ b/openmdao/core/constants.py @@ -128,3 +128,10 @@ def __deepcopy__(self, memo): # Use this as a special value to be able to tell if the caller set a value for the optional # out_stream argument. We run into problems running testflo if we use a default of sys.stdout. _DEFAULT_OUT_STREAM = _ReprClass("DEFAULT_OUT_STREAM") + +# Use in pyOptSparseDriver. The default is the reports directory which includes the directory +# named after the Problem name. But when the declare method in that class is called, the driver +# does not have a reference to the Problem so can't get the name. This serves as a flag that +# the default directory in the reports directory is what is wanted. Then in the run method, +# the actual default directory is used +_DEFAULT_PYOPT_SPARSE_OUTPUT_DIR = _ReprClass("DEFAULT_PYOPT_SPARSE_OUTPUT_DIR") diff --git a/openmdao/core/problem.py b/openmdao/core/problem.py index 84345c42f9..e721376984 100644 --- a/openmdao/core/problem.py +++ b/openmdao/core/problem.py @@ -1,6 +1,7 @@ """Define the Problem class and a FakeComm class for non-MPI users.""" import __main__ +import shutil import sys import pprint @@ -349,6 +350,11 @@ def __init__(self, model=None, driver=None, comm=None, name=None, reports=_UNDEF desc='Patterns for vars to exclude in recording ' '(processed post-includes). Uses fnmatch wildcards') + # Start a run by deleting any existing reports so that the files + # that are in that directory are all from this run and not a previous run + if os.path.isdir(get_reports_dir()): + shutil.rmtree(get_reports_dir()) + # register hooks for any reports activate_reports(self._reports, self) diff --git a/openmdao/drivers/pyoptsparse_driver.py b/openmdao/drivers/pyoptsparse_driver.py index 4219e349fe..8a482642f2 100644 --- a/openmdao/drivers/pyoptsparse_driver.py +++ b/openmdao/drivers/pyoptsparse_driver.py @@ -5,7 +5,7 @@ formulating and solving nonlinear constrained optimization problems, with additional MPI capability. """ - +import pathlib import sys import json import signal @@ -22,12 +22,13 @@ except Exception as err: pyoptsparse = err -from openmdao.core.constants import INT_DTYPE +from openmdao.core.constants import INT_DTYPE, _DEFAULT_PYOPT_SPARSE_OUTPUT_DIR, _ReprClass from openmdao.core.analysis_error import AnalysisError from openmdao.core.driver import Driver, RecordingDebugging from openmdao.utils.class_util import WeakMethodWrapper from openmdao.utils.mpi import FakeComm, MPI from openmdao.utils.om_warnings import issue_warning, warn_deprecation +from openmdao.utils.reports_system import get_reports_dir # what version of pyoptspare are we working with if pyoptsparse and hasattr(pyoptsparse, '__version__'): @@ -268,6 +269,10 @@ def _declare_options(self): self.options.declare('hotstart_file', types=str, default=None, allow_none=True, desc='File location of a pyopt_sparse optimization history to use ' 'to hot start the optimization. Default is None.') + self.options.declare('output_dir', types=(str,_ReprClass), default=_DEFAULT_PYOPT_SPARSE_OUTPUT_DIR, allow_none=True, + desc='Directory location of pyopt_sparse output files.' + 'Default is ./reports_directory/problem_name.') + @property def hist_file(self): @@ -373,6 +378,47 @@ def run(self): self._quantities = [] optimizer = self.options['optimizer'] + + # Need to tell optimizer where to put its .out files + if self.options['output_dir'] is None: + self.options['output_dir'] = "." + elif self.options['output_dir'] == _DEFAULT_PYOPT_SPARSE_OUTPUT_DIR: + problem = self._problem() + default_output_dir = pathlib.Path(get_reports_dir()).joinpath(problem._name) + pathlib.Path(default_output_dir).mkdir(parents=True, exist_ok=True) + self.options['output_dir'] = str(default_output_dir) + output_dir = self.options['output_dir'] + + if optimizer == 'ALPSO': # Actually, this is the root of two files generated + if problem.driver.opt_settings.get('filename') is None: + problem.driver.opt_settings['filename'] = f'{output_dir}/ALPSO.out' + elif optimizer == 'CONMIN': + if problem.driver.opt_settings.get('IFILE') is None: + problem.driver.opt_settings['IFILE'] = f'{output_dir}/CONMIN.out' + elif optimizer == 'IPOPT': + if problem.driver.opt_settings.get('output_file') is None: + problem.driver.opt_settings['output_file'] = f'{output_dir}/IPOPT.out' + elif optimizer == 'NLPQLP': + if problem.driver.opt_settings.get('iFile') is None: + problem.driver.opt_settings['iFile'] = f'{output_dir}/NLPQLP.out' + # Nothing for NSGA2 + elif optimizer == 'PSQP': + if problem.driver.opt_settings.get('IFILE') is None: + problem.driver.opt_settings['IFILE'] = f'{output_dir}/PSQP.out' + elif optimizer == 'PAROPT': + if problem.driver.opt_settings.get('tr_output_file') is None: + problem.driver.opt_settings['tr_output_file'] = f'{output_dir}/paropt.tr' + if problem.driver.opt_settings.get('output_file') is None: + problem.driver.opt_settings['output_file'] = f'{output_dir}/paropt.out' + elif optimizer == 'SLSQP': + if problem.driver.opt_settings.get('IFILE') is None: + problem.driver.opt_settings['IFILE'] = f'{output_dir}/SLSQP.out' + elif optimizer == 'SNOPT': + if problem.driver.opt_settings.get('Print file') is None: + problem.driver.opt_settings['Print file'] = f'{output_dir}/SNOPT_print.out' + if problem.driver.opt_settings.get('Summary file') is None: + problem.driver.opt_settings['Summary file'] = f'{output_dir}/SNOPT_summary.out' + self._fill_NANs = not respects_fail_flag[self.options['optimizer']] self._check_for_missing_objective() diff --git a/openmdao/drivers/tests/test_pyoptsparse_driver.py b/openmdao/drivers/tests/test_pyoptsparse_driver.py index ffc4bcdce7..9d79b53d8f 100644 --- a/openmdao/drivers/tests/test_pyoptsparse_driver.py +++ b/openmdao/drivers/tests/test_pyoptsparse_driver.py @@ -1,6 +1,7 @@ """ Unit tests for the Pyoptsparse Driver.""" import copy +import pathlib import unittest import os.path @@ -16,6 +17,7 @@ from openmdao.test_suite.components.sellar import SellarDerivativesGrouped from openmdao.utils.assert_utils import assert_near_equal, assert_warning, assert_check_totals from openmdao.utils.general_utils import set_pyoptsparse_opt, run_driver +from openmdao.utils.reports_system import get_reports_dir from openmdao.utils.testing_utils import use_tempdirs, require_pyoptsparse from openmdao.utils.om_warnings import OMDeprecationWarning from openmdao.utils.mpi import MPI @@ -3255,6 +3257,58 @@ def test_resize(self): p.run_driver() p.compute_totals() +@unittest.skipIf(OPT is None or OPTIMIZER is None, "only run if pyoptsparse is installed.") +@use_tempdirs +class TestPyoptSparseOutputFiles(unittest.TestCase): + + def run_and_test(self, optimizer, output_file_names): + prob = self.createParaboloidProblem() + prob.driver = pyOptSparseDriver(optimizer=optimizer, print_results=False) + + if optimizer == 'ALPSO': + prob.driver.opt_settings['fileout'] = 3 + + prob.run_driver() + default_output_dir = pathlib.Path(get_reports_dir()).joinpath(prob._name) + for opt_setting_name, output_file_name in output_file_names: + output_file = default_output_dir.joinpath(output_file_name) + self.assertTrue(output_file.is_file(), + f"{output_file_name} output file not found at {str(output_file)}") + + def test_default_output_dir_default(self): + optimizers_and_output_files = { + 'ALPSO': [('filename','ALPSO_print.out'), ('filename','ALPSO_summary.out')], + 'CONMIN': [('IFILE', 'CONMIN.out')], + 'IPOPT': [('output_file', 'IPOPT.out')], + 'NLPQLP': [('iFile', 'NLPQLP.out')], + 'PSQP': [('IFILE', 'PSQP.out')], + 'PAROPT': [('tr_output_file', 'paropt.tr'), ('output_file', 'paropt.out')], + 'SLSQP': [('IFILE', 'SLSQP.out')], + 'SNOPT': [('Print file', 'SNOPT_print.out'), ('Summary file', 'SNOPT_summary.out')] + } + for optimizer, output_files in optimizers_and_output_files.items(): + _, loc_opt = set_pyoptsparse_opt(optimizer) + if loc_opt == optimizer: # Only do optimizers that are installed + self.run_and_test(optimizer, output_files) + # self.run_and_test('CONMIN', ['CONMIN.out']) + + def createParaboloidProblem(self): + prob = om.Problem() + model = prob.model + + model.add_subsystem('p1', om.IndepVarComp('x', 50.0), promotes=['*']) + model.add_subsystem('p2', om.IndepVarComp('y', 50.0), promotes=['*']) + model.add_subsystem('comp', Paraboloid(), promotes=['*']) + model.add_subsystem('con', om.ExecComp('c = - x + y'), promotes=['*']) + prob.set_solver_print(level=0) + model.add_design_var('x', lower=-50.0, upper=50.0) + model.add_design_var('y', lower=-50.0, upper=50.0) + model.add_objective('f_xy') + model.add_constraint('c', upper=-15.0) + prob.setup() + return prob + + if __name__ == "__main__": unittest.main() From 89053d816a8a88431f241f3b1ced413db102ae35 Mon Sep 17 00:00:00 2001 From: Herb Schilling Date: Mon, 6 Nov 2023 16:33:01 -0500 Subject: [PATCH 02/22] Added tests for when user sets explicit locations for optimizer output files and also when they set it to None to indicate they want the files in the directory the script is run from --- .../drivers/tests/test_pyoptsparse_driver.py | 109 +++++++++++++----- 1 file changed, 81 insertions(+), 28 deletions(-) diff --git a/openmdao/drivers/tests/test_pyoptsparse_driver.py b/openmdao/drivers/tests/test_pyoptsparse_driver.py index 9d79b53d8f..a3c21857f5 100644 --- a/openmdao/drivers/tests/test_pyoptsparse_driver.py +++ b/openmdao/drivers/tests/test_pyoptsparse_driver.py @@ -3260,13 +3260,38 @@ def test_resize(self): @unittest.skipIf(OPT is None or OPTIMIZER is None, "only run if pyoptsparse is installed.") @use_tempdirs class TestPyoptSparseOutputFiles(unittest.TestCase): + optimizers_and_output_files = { + 'ALPSO': [('filename', 'ALPSO_print.out'), ('filename', 'ALPSO_summary.out')], + 'CONMIN': [('IFILE', 'CONMIN.out')], + 'IPOPT': [('output_file', 'IPOPT.out')], + 'NLPQLP': [('iFile', 'NLPQLP.out')], + 'PSQP': [('IFILE', 'PSQP.out')], + 'PAROPT': [('tr_output_file', 'paropt.tr'), ('output_file', 'paropt.out')], + 'SLSQP': [('IFILE', 'SLSQP.out')], + 'SNOPT': [('Print file', 'SNOPT_print.out'), ('Summary file', 'SNOPT_summary.out')] + } + + def createParaboloidProblem(self): + prob = om.Problem() + model = prob.model + model.add_subsystem('p1', om.IndepVarComp('x', 50.0), promotes=['*']) + model.add_subsystem('p2', om.IndepVarComp('y', 50.0), promotes=['*']) + model.add_subsystem('comp', Paraboloid(), promotes=['*']) + model.add_subsystem('con', om.ExecComp('c = - x + y'), promotes=['*']) + prob.set_solver_print(level=0) + model.add_design_var('x', lower=-50.0, upper=50.0) + model.add_design_var('y', lower=-50.0, upper=50.0) + model.add_objective('f_xy') + model.add_constraint('c', upper=-15.0) + prob.setup() + return prob def run_and_test(self, optimizer, output_file_names): prob = self.createParaboloidProblem() prob.driver = pyOptSparseDriver(optimizer=optimizer, print_results=False) if optimizer == 'ALPSO': - prob.driver.opt_settings['fileout'] = 3 + prob.driver.opt_settings['fileout'] = 3 # need this to be 3 to get the output files prob.run_driver() default_output_dir = pathlib.Path(get_reports_dir()).joinpath(prob._name) @@ -3275,38 +3300,66 @@ def run_and_test(self, optimizer, output_file_names): self.assertTrue(output_file.is_file(), f"{output_file_name} output file not found at {str(output_file)}") - def test_default_output_dir_default(self): - optimizers_and_output_files = { - 'ALPSO': [('filename','ALPSO_print.out'), ('filename','ALPSO_summary.out')], - 'CONMIN': [('IFILE', 'CONMIN.out')], - 'IPOPT': [('output_file', 'IPOPT.out')], - 'NLPQLP': [('iFile', 'NLPQLP.out')], - 'PSQP': [('IFILE', 'PSQP.out')], - 'PAROPT': [('tr_output_file', 'paropt.tr'), ('output_file', 'paropt.out')], - 'SLSQP': [('IFILE', 'SLSQP.out')], - 'SNOPT': [('Print file', 'SNOPT_print.out'), ('Summary file', 'SNOPT_summary.out')] - } - for optimizer, output_files in optimizers_and_output_files.items(): + def run_and_test_previous_behavior(self, optimizer, output_file_names): + # output_file_names is a list of tuples + prob = self.createParaboloidProblem() + prob.driver = pyOptSparseDriver(optimizer=optimizer, print_results=False) + + if optimizer == 'ALPSO': + prob.driver.opt_settings['fileout'] = 3 + + prob.driver.options['output_dir'] = None + + prob.run_driver() + for opt_setting_name, output_file_name in output_file_names: + output_file = pathlib.Path(output_file_name) + self.assertTrue(output_file.is_file(), + f"{output_file_name} output file not found at {str(output_file)}") + + def run_and_test_user_set(self, optimizer, output_file_names): + # output_file_names is a list of tuples of setting name and output file name + + user_directory_name = 'user_reports_dir' + pathlib.Path(user_directory_name).mkdir(exist_ok=True) + + prob = self.createParaboloidProblem() + prob.driver = pyOptSparseDriver(optimizer=optimizer, print_results=False) + + if optimizer == 'ALPSO': + prob.driver.opt_settings['fileout'] = 3 + + for opt_setting_name, output_file_name in output_file_names: + output_file_path = pathlib.Path(user_directory_name).joinpath(output_file_name) + if optimizer == 'ALPSO': # Since ALPSO only has the one setting that affects both + # output files + output_file_path = pathlib.Path(user_directory_name).joinpath('ALPSO.out') + prob.driver.opt_settings[opt_setting_name] = str(output_file_path) + + prob.run_driver() + + for opt_setting_name, output_file_name in output_file_names: + output_file_path = pathlib.Path(user_directory_name).joinpath(output_file_name) + self.assertTrue(output_file_path.is_file(), + f"{str(output_file_path)} output file not found at {str(output_file_path)}") + + def test_default_output_dir(self): + for optimizer, output_files in self.optimizers_and_output_files.items(): _, loc_opt = set_pyoptsparse_opt(optimizer) if loc_opt == optimizer: # Only do optimizers that are installed self.run_and_test(optimizer, output_files) - # self.run_and_test('CONMIN', ['CONMIN.out']) - def createParaboloidProblem(self): - prob = om.Problem() - model = prob.model + def test_previous_behavior_output_dir(self): + for optimizer, output_files in self.optimizers_and_output_files.items(): + _, loc_opt = set_pyoptsparse_opt(optimizer) + if loc_opt == optimizer: # Only do optimizers that are installed + self.run_and_test(optimizer, output_files) + + def test_user_set_output_dir(self): + for optimizer, output_files in self.optimizers_and_output_files.items(): + _, loc_opt = set_pyoptsparse_opt(optimizer) + if loc_opt == optimizer: # Only do optimizers that are installed + self.run_and_test_user_set(optimizer, output_files) - model.add_subsystem('p1', om.IndepVarComp('x', 50.0), promotes=['*']) - model.add_subsystem('p2', om.IndepVarComp('y', 50.0), promotes=['*']) - model.add_subsystem('comp', Paraboloid(), promotes=['*']) - model.add_subsystem('con', om.ExecComp('c = - x + y'), promotes=['*']) - prob.set_solver_print(level=0) - model.add_design_var('x', lower=-50.0, upper=50.0) - model.add_design_var('y', lower=-50.0, upper=50.0) - model.add_objective('f_xy') - model.add_constraint('c', upper=-15.0) - prob.setup() - return prob From 85b09b7ce21ac3b77f6b7b25a9ce0533956d766b Mon Sep 17 00:00:00 2001 From: Herb Schilling Date: Tue, 7 Nov 2023 13:52:50 -0500 Subject: [PATCH 03/22] Removed %px which was not needed for that cell --- .../adding_desvars_cons_objs/adding_constraint.ipynb | 1 - 1 file changed, 1 deletion(-) diff --git a/openmdao/docs/openmdao_book/features/core_features/adding_desvars_cons_objs/adding_constraint.ipynb b/openmdao/docs/openmdao_book/features/core_features/adding_desvars_cons_objs/adding_constraint.ipynb index f2d57e7011..1d7fa5c19e 100644 --- a/openmdao/docs/openmdao_book/features/core_features/adding_desvars_cons_objs/adding_constraint.ipynb +++ b/openmdao/docs/openmdao_book/features/core_features/adding_desvars_cons_objs/adding_constraint.ipynb @@ -260,7 +260,6 @@ }, "outputs": [], "source": [ - "%%px\n", "import openmdao.api as om\n", "om.display_source(\"openmdao.test_suite.components.paraboloid_distributed.DistParabFeature\")" ] From a5ebafe6f92f95dadb9d58c5862900071346c7c4 Mon Sep 17 00:00:00 2001 From: Herb Schilling Date: Wed, 15 Nov 2023 11:14:03 -0500 Subject: [PATCH 04/22] Added another constant to indicate using a default value. Used in options declare calls. At the point of the call, the value for the default has not been created yet --- openmdao/core/constants.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/openmdao/core/constants.py b/openmdao/core/constants.py index ed1ff954c1..c4c987bad9 100644 --- a/openmdao/core/constants.py +++ b/openmdao/core/constants.py @@ -129,9 +129,9 @@ def __deepcopy__(self, memo): # out_stream argument. We run into problems running testflo if we use a default of sys.stdout. _DEFAULT_OUT_STREAM = _ReprClass("DEFAULT_OUT_STREAM") -# Use in pyOptSparseDriver. The default is the reports directory which includes the directory +# Used in pyOptSparseDriver and Problem for coloring dir. The default is the reports directory which includes the directory # named after the Problem name. But when the declare method in that class is called, the driver # does not have a reference to the Problem so can't get the name. This serves as a flag that # the default directory in the reports directory is what is wanted. Then in the run method, # the actual default directory is used -_DEFAULT_PYOPT_SPARSE_OUTPUT_DIR = _ReprClass("DEFAULT_PYOPT_SPARSE_OUTPUT_DIR") +_DEFAULT_REPORTS_DIR = _ReprClass("DEFAULT_PYOPT_SPARSE_OUTPUT_DIR") From 0dc63f2dfa21317909ea18b231189112fe9b3f53 Mon Sep 17 00:00:00 2001 From: Herb Schilling Date: Wed, 15 Nov 2023 11:15:25 -0500 Subject: [PATCH 05/22] Corrected a typo. --- openmdao/core/tests/test_partial_color.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openmdao/core/tests/test_partial_color.py b/openmdao/core/tests/test_partial_color.py index 9782be3404..41cf13f7f0 100644 --- a/openmdao/core/tests/test_partial_color.py +++ b/openmdao/core/tests/test_partial_color.py @@ -1867,4 +1867,4 @@ def test_simple_partials_explicit(self, method): if __name__ == '__main__': - unitest.main() + unittest.main() From 8ad056ed136b3ea76acb35d995aa3a823c9116c1 Mon Sep 17 00:00:00 2001 From: Herb Schilling Date: Wed, 15 Nov 2023 11:23:46 -0500 Subject: [PATCH 06/22] Added tests for the new feature where by default the coloring files go in the reports directory --- openmdao/core/tests/test_coloring.py | 49 +++++++++++++++++++++++++++- 1 file changed, 48 insertions(+), 1 deletion(-) diff --git a/openmdao/core/tests/test_coloring.py b/openmdao/core/tests/test_coloring.py index 0538732a5a..5da0cca76d 100644 --- a/openmdao/core/tests/test_coloring.py +++ b/openmdao/core/tests/test_coloring.py @@ -1,5 +1,6 @@ import os +import pathlib import sys import itertools import pickle @@ -10,6 +11,9 @@ from io import StringIO from numpy.testing import assert_almost_equal + +from openmdao.utils.reports_system import get_reports_dir + try: from scipy.sparse import load_npz except ImportError: @@ -89,7 +93,9 @@ def compute(self, inputs, outputs): def run_opt(driver_class, mode, assemble_type=None, color_info=None, derivs=True, recorder=None, has_lin_constraint=True, has_diag_partials=True, partial_coloring=False, - use_vois=True, auto_ivc=False, con_alias=False, check=False, **options): + use_vois=True, auto_ivc=False, con_alias=False, check=False, + problem_options=None, + **options): p = om.Problem(model=CounterGroup()) @@ -252,6 +258,9 @@ def compute(self, inputs, outputs): if recorder: p.driver.add_recorder(recorder) + if 'coloring_dir' in problem_options: + p.options['coloring_dir'] = problem_options['coloring_dir'] + p.setup(mode=mode, derivatives=derivs, check=check) if use_vois: p.run_driver() @@ -1529,5 +1538,43 @@ def test_wrong_pickle(self): self.assertEqual(ctx.exception.args[0], "File '_bad_pickle_' is not a valid coloring file.") + +@use_tempdirs +class TestSettingColoringDir(unittest.TestCase): + + def test_coloring_dir_is_None(self): + problem_options = { + 'coloring_dir': None, + } + p = run_opt(pyOptSparseDriver, 'auto', assemble_type='csc', optimizer='SLSQP', + dynamic_total_coloring=True, print_results=False, + problem_options=problem_options) + + coloring_dir = pathlib.Path('./coloring_files') + self.assertTrue(coloring_dir.is_dir(), 'Coloring dir not found') + + def test_coloring_dir_is_custom(self): + problem_options = { + 'coloring_dir': 'custom_coloring_dir', + } + p = run_opt(pyOptSparseDriver, 'auto', assemble_type='csc', optimizer='SLSQP', + dynamic_total_coloring=True, print_results=False, problem_options=problem_options) + + coloring_dir = pathlib.Path('custom_coloring_dir') + self.assertTrue(coloring_dir.is_dir(), 'Coloring dir not found') + + def test_coloring_dir_is_default(self): + problem_options = { + } + p = run_opt(pyOptSparseDriver, 'auto', assemble_type='csc', optimizer='SLSQP', + dynamic_total_coloring=True, print_results=False, + problem_options=problem_options) + + coloring_dir = pathlib.Path(get_reports_dir()).joinpath(p._name).joinpath( + 'coloring_files') + self.assertTrue(coloring_dir.is_dir(), 'Coloring dir not found') + + + if __name__ == '__main__': unittest.main() From 73ae831dba0c3a5ef36c42119524b71c4860045a Mon Sep 17 00:00:00 2001 From: Herb Schilling Date: Wed, 15 Nov 2023 14:25:00 -0500 Subject: [PATCH 07/22] Changed the handling of the coloring_dir option to match our new thinking on that. Also now removing the reports dir before the run to have a clean directory --- openmdao/core/problem.py | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/openmdao/core/problem.py b/openmdao/core/problem.py index e721376984..21a6bf7dc6 100644 --- a/openmdao/core/problem.py +++ b/openmdao/core/problem.py @@ -19,7 +19,8 @@ import numpy as np import scipy.sparse as sparse -from openmdao.core.constants import _SetupStatus +from openmdao.core.constants import _SetupStatus, _DEFAULT_REPORTS_DIR, _ReprClass + from openmdao.core.component import Component from openmdao.core.driver import Driver, record_iteration, SaveOptResult from openmdao.core.explicitcomponent import ExplicitComponent @@ -287,9 +288,10 @@ def __init__(self, model=None, driver=None, comm=None, name=None, reports=_UNDEF # General options self.options = OptionsDictionary(parent_name=type(self).__name__) - self.options.declare('coloring_dir', types=str, - default=os.path.join(os.getcwd(), 'coloring_files'), - desc='Directory containing coloring files (if any) for this Problem.') + self.options.declare('coloring_dir', types=(str,_ReprClass), default=_DEFAULT_REPORTS_DIR, + allow_none=True, + desc='Directory containing coloring files (if any) for this Problem.' + 'Default is ./reports_directory/problem_name/coloring_dir.') self.options.declare('group_by_pre_opt_post', types=bool, default=False, desc="If True, group subsystems of the top level model into " @@ -968,6 +970,17 @@ def setup(self, check=False, logger=None, mode='auto', force_alloc_complex=False model_comm = self.driver._setup_comm(comm) + # set the coloring_dir option based on user settings. + # By default, put it in the reports dir + # If set to None, put it in the current directory (the previous behavior) + # Otherwise, use what ever the user set the directory to + if self.options['coloring_dir'] is None: + self.options['coloring_dir'] = "./coloring_files" + elif self.options['coloring_dir'] == _DEFAULT_REPORTS_DIR: + default_coloring_dir = pathlib.Path(get_reports_dir()).joinpath(self._name).joinpath('coloring_files') + pathlib.Path(default_coloring_dir).mkdir(parents=True, exist_ok=True) + self.options['coloring_dir'] = str(default_coloring_dir) + # this metadata will be shared by all Systems/Solvers in the system tree self._metadata = { 'name': self._name, # the name of this Problem From 5c0a4118a44c44b80103c026eedd2f4263e79982 Mon Sep 17 00:00:00 2001 From: Herb Schilling Date: Wed, 15 Nov 2023 14:26:52 -0500 Subject: [PATCH 08/22] Added output_dir option to set where to put optimizer output files --- openmdao/drivers/pyoptsparse_driver.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/openmdao/drivers/pyoptsparse_driver.py b/openmdao/drivers/pyoptsparse_driver.py index 8a482642f2..9a8a4832d4 100644 --- a/openmdao/drivers/pyoptsparse_driver.py +++ b/openmdao/drivers/pyoptsparse_driver.py @@ -22,7 +22,7 @@ except Exception as err: pyoptsparse = err -from openmdao.core.constants import INT_DTYPE, _DEFAULT_PYOPT_SPARSE_OUTPUT_DIR, _ReprClass +from openmdao.core.constants import INT_DTYPE, _DEFAULT_REPORTS_DIR, _ReprClass from openmdao.core.analysis_error import AnalysisError from openmdao.core.driver import Driver, RecordingDebugging from openmdao.utils.class_util import WeakMethodWrapper @@ -269,7 +269,7 @@ def _declare_options(self): self.options.declare('hotstart_file', types=str, default=None, allow_none=True, desc='File location of a pyopt_sparse optimization history to use ' 'to hot start the optimization. Default is None.') - self.options.declare('output_dir', types=(str,_ReprClass), default=_DEFAULT_PYOPT_SPARSE_OUTPUT_DIR, allow_none=True, + self.options.declare('output_dir', types=(str,_ReprClass), default=_DEFAULT_REPORTS_DIR, allow_none=True, desc='Directory location of pyopt_sparse output files.' 'Default is ./reports_directory/problem_name.') @@ -382,7 +382,7 @@ def run(self): # Need to tell optimizer where to put its .out files if self.options['output_dir'] is None: self.options['output_dir'] = "." - elif self.options['output_dir'] == _DEFAULT_PYOPT_SPARSE_OUTPUT_DIR: + elif self.options['output_dir'] == _DEFAULT_REPORTS_DIR: problem = self._problem() default_output_dir = pathlib.Path(get_reports_dir()).joinpath(problem._name) pathlib.Path(default_output_dir).mkdir(parents=True, exist_ok=True) From cd7c1c0e458beb84b64fcf54758b7d0e17f840ed Mon Sep 17 00:00:00 2001 From: Herb Schilling Date: Wed, 15 Nov 2023 14:27:20 -0500 Subject: [PATCH 09/22] Added tests for the new output_dir option on pyoptsparse --- .../drivers/tests/test_pyoptsparse_driver.py | 22 ++++++++++++------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/openmdao/drivers/tests/test_pyoptsparse_driver.py b/openmdao/drivers/tests/test_pyoptsparse_driver.py index a3c21857f5..b4c6e264f8 100644 --- a/openmdao/drivers/tests/test_pyoptsparse_driver.py +++ b/openmdao/drivers/tests/test_pyoptsparse_driver.py @@ -3260,7 +3260,9 @@ def test_resize(self): @unittest.skipIf(OPT is None or OPTIMIZER is None, "only run if pyoptsparse is installed.") @use_tempdirs class TestPyoptSparseOutputFiles(unittest.TestCase): + # dict of optimizers with the values being a list of tuples of ( option_name, output_file_name ) optimizers_and_output_files = { + # ALPSO uses a single option `filename` to determine name of both output files 'ALPSO': [('filename', 'ALPSO_print.out'), ('filename', 'ALPSO_summary.out')], 'CONMIN': [('IFILE', 'CONMIN.out')], 'IPOPT': [('output_file', 'IPOPT.out')], @@ -3286,7 +3288,9 @@ def createParaboloidProblem(self): prob.setup() return prob - def run_and_test(self, optimizer, output_file_names): + def run_and_test_default_output_dir(self, optimizer, output_file_names): + # default is to put the files in the reports directory under the problem name folder + # output_file_names is a list of tuples of setting name and output file name prob = self.createParaboloidProblem() prob.driver = pyOptSparseDriver(optimizer=optimizer, print_results=False) @@ -3300,8 +3304,9 @@ def run_and_test(self, optimizer, output_file_names): self.assertTrue(output_file.is_file(), f"{output_file_name} output file not found at {str(output_file)}") - def run_and_test_previous_behavior(self, optimizer, output_file_names): - # output_file_names is a list of tuples + def run_and_test_previous_behavior_output_dir(self, optimizer, output_file_names): + # Previous behavior is to put files in current working directory. Indicated with None + # output_file_names is a list of tuples of setting name and output file name prob = self.createParaboloidProblem() prob.driver = pyOptSparseDriver(optimizer=optimizer, print_results=False) @@ -3316,7 +3321,7 @@ def run_and_test_previous_behavior(self, optimizer, output_file_names): self.assertTrue(output_file.is_file(), f"{output_file_name} output file not found at {str(output_file)}") - def run_and_test_user_set(self, optimizer, output_file_names): + def run_and_test_user_set_output_dir(self, optimizer, output_file_names): # output_file_names is a list of tuples of setting name and output file name user_directory_name = 'user_reports_dir' @@ -3329,10 +3334,11 @@ def run_and_test_user_set(self, optimizer, output_file_names): prob.driver.opt_settings['fileout'] = 3 for opt_setting_name, output_file_name in output_file_names: - output_file_path = pathlib.Path(user_directory_name).joinpath(output_file_name) if optimizer == 'ALPSO': # Since ALPSO only has the one setting that affects both # output files output_file_path = pathlib.Path(user_directory_name).joinpath('ALPSO.out') + else: + output_file_path = pathlib.Path(user_directory_name).joinpath(output_file_name) prob.driver.opt_settings[opt_setting_name] = str(output_file_path) prob.run_driver() @@ -3346,19 +3352,19 @@ def test_default_output_dir(self): for optimizer, output_files in self.optimizers_and_output_files.items(): _, loc_opt = set_pyoptsparse_opt(optimizer) if loc_opt == optimizer: # Only do optimizers that are installed - self.run_and_test(optimizer, output_files) + self.run_and_test_default_output_dir(optimizer, output_files) def test_previous_behavior_output_dir(self): for optimizer, output_files in self.optimizers_and_output_files.items(): _, loc_opt = set_pyoptsparse_opt(optimizer) if loc_opt == optimizer: # Only do optimizers that are installed - self.run_and_test(optimizer, output_files) + self.run_and_test_previous_behavior_output_dir(optimizer, output_files) def test_user_set_output_dir(self): for optimizer, output_files in self.optimizers_and_output_files.items(): _, loc_opt = set_pyoptsparse_opt(optimizer) if loc_opt == optimizer: # Only do optimizers that are installed - self.run_and_test_user_set(optimizer, output_files) + self.run_and_test_user_set_output_dir(optimizer, output_files) From 4cf2290b0f947d26b79a2f10e7d0518888535287 Mon Sep 17 00:00:00 2001 From: Herb Schilling Date: Wed, 15 Nov 2023 15:39:53 -0500 Subject: [PATCH 10/22] Changed location of the optimizer output files to be where this test assumed they would be when this test was written:in the current working directory --- openmdao/drivers/tests/test_analysis_errors.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/openmdao/drivers/tests/test_analysis_errors.py b/openmdao/drivers/tests/test_analysis_errors.py index f782eda4c9..569dcf83c6 100644 --- a/openmdao/drivers/tests/test_analysis_errors.py +++ b/openmdao/drivers/tests/test_analysis_errors.py @@ -111,6 +111,8 @@ def setup_problem(self, optimizer, func=None): if optimizer in self.opt_settings: driver.opt_settings = self.opt_settings[optimizer] driver.options['print_results'] = False + driver.options['output_dir'] = None # So output goes in current working directory + # that was the location when this test was written # setup problem & initialize values prob = om.Problem(model, driver) From a98d6cbb6748faa2c8c0252889052c710386048f Mon Sep 17 00:00:00 2001 From: Herb Schilling Date: Mon, 20 Nov 2023 16:15:30 -0500 Subject: [PATCH 11/22] All but some coloring tests pass. Those tests need a decision on how to handle coloring pickle files. Waiting for team members to come back from leave --- openmdao/core/problem.py | 9 ++++++--- openmdao/core/tests/test_coloring.py | 2 +- openmdao/utils/tests/test_reports_system.py | 1 + 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/openmdao/core/problem.py b/openmdao/core/problem.py index 21a6bf7dc6..af8f3e6fcf 100644 --- a/openmdao/core/problem.py +++ b/openmdao/core/problem.py @@ -354,8 +354,10 @@ def __init__(self, model=None, driver=None, comm=None, name=None, reports=_UNDEF # Start a run by deleting any existing reports so that the files # that are in that directory are all from this run and not a previous run - if os.path.isdir(get_reports_dir()): - shutil.rmtree(get_reports_dir()) + reports_dirpath = pathlib.Path(get_reports_dir()).joinpath(f'{self._name}') + if self.comm.rank == 0: + if os.path.isdir(reports_dirpath): + shutil.rmtree(reports_dirpath) # register hooks for any reports activate_reports(self._reports, self) @@ -978,8 +980,9 @@ def setup(self, check=False, logger=None, mode='auto', force_alloc_complex=False self.options['coloring_dir'] = "./coloring_files" elif self.options['coloring_dir'] == _DEFAULT_REPORTS_DIR: default_coloring_dir = pathlib.Path(get_reports_dir()).joinpath(self._name).joinpath('coloring_files') - pathlib.Path(default_coloring_dir).mkdir(parents=True, exist_ok=True) self.options['coloring_dir'] = str(default_coloring_dir) + if 'total_coloring' in self._reports: + pathlib.Path(self.options['coloring_dir']).mkdir(parents=True, exist_ok=True) # this metadata will be shared by all Systems/Solvers in the system tree self._metadata = { diff --git a/openmdao/core/tests/test_coloring.py b/openmdao/core/tests/test_coloring.py index 5da0cca76d..1298775afb 100644 --- a/openmdao/core/tests/test_coloring.py +++ b/openmdao/core/tests/test_coloring.py @@ -258,7 +258,7 @@ def compute(self, inputs, outputs): if recorder: p.driver.add_recorder(recorder) - if 'coloring_dir' in problem_options: + if problem_options and 'coloring_dir' in problem_options: p.options['coloring_dir'] = problem_options['coloring_dir'] p.setup(mode=mode, derivatives=derivs, check=check) diff --git a/openmdao/utils/tests/test_reports_system.py b/openmdao/utils/tests/test_reports_system.py index a4501c1085..433b2586a0 100644 --- a/openmdao/utils/tests/test_reports_system.py +++ b/openmdao/utils/tests/test_reports_system.py @@ -135,6 +135,7 @@ def solve(self): return super().solve() + prob = om.Problem(reports=prob1_reports) prob.model.add_subsystem('indep', om.IndepVarComp('x', 1.0)) G = prob.model.add_subsystem('G', om.Group()) From 7542ae9b2b11c248b71a00df9884d68cf2a2bb6d Mon Sep 17 00:00:00 2001 From: Herb Schilling Date: Tue, 21 Nov 2023 10:12:41 -0500 Subject: [PATCH 12/22] pep8 fix --- openmdao/core/constants.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/openmdao/core/constants.py b/openmdao/core/constants.py index c4c987bad9..cea5f833a5 100644 --- a/openmdao/core/constants.py +++ b/openmdao/core/constants.py @@ -129,7 +129,8 @@ def __deepcopy__(self, memo): # out_stream argument. We run into problems running testflo if we use a default of sys.stdout. _DEFAULT_OUT_STREAM = _ReprClass("DEFAULT_OUT_STREAM") -# Used in pyOptSparseDriver and Problem for coloring dir. The default is the reports directory which includes the directory +# Used in pyOptSparseDriver and Problem for coloring dir. The default is the reports directory +# which includes the directory # named after the Problem name. But when the declare method in that class is called, the driver # does not have a reference to the Problem so can't get the name. This serves as a flag that # the default directory in the reports directory is what is wanted. Then in the run method, From 2f43b832748155e3956770a659e01d245b2cde9e Mon Sep 17 00:00:00 2001 From: Herb Schilling Date: Tue, 21 Nov 2023 10:15:05 -0500 Subject: [PATCH 13/22] removed moving coloring file for now --- openmdao/core/problem.py | 19 +++----------- openmdao/core/tests/test_coloring.py | 39 ---------------------------- 2 files changed, 3 insertions(+), 55 deletions(-) diff --git a/openmdao/core/problem.py b/openmdao/core/problem.py index af8f3e6fcf..ae5ab7c51a 100644 --- a/openmdao/core/problem.py +++ b/openmdao/core/problem.py @@ -288,10 +288,9 @@ def __init__(self, model=None, driver=None, comm=None, name=None, reports=_UNDEF # General options self.options = OptionsDictionary(parent_name=type(self).__name__) - self.options.declare('coloring_dir', types=(str,_ReprClass), default=_DEFAULT_REPORTS_DIR, - allow_none=True, - desc='Directory containing coloring files (if any) for this Problem.' - 'Default is ./reports_directory/problem_name/coloring_dir.') + self.options.declare('coloring_dir', types=str, + default=os.path.join(os.getcwd(), 'coloring_files'), + desc='Directory containing coloring files (if any) for this Problem.') self.options.declare('group_by_pre_opt_post', types=bool, default=False, desc="If True, group subsystems of the top level model into " @@ -972,18 +971,6 @@ def setup(self, check=False, logger=None, mode='auto', force_alloc_complex=False model_comm = self.driver._setup_comm(comm) - # set the coloring_dir option based on user settings. - # By default, put it in the reports dir - # If set to None, put it in the current directory (the previous behavior) - # Otherwise, use what ever the user set the directory to - if self.options['coloring_dir'] is None: - self.options['coloring_dir'] = "./coloring_files" - elif self.options['coloring_dir'] == _DEFAULT_REPORTS_DIR: - default_coloring_dir = pathlib.Path(get_reports_dir()).joinpath(self._name).joinpath('coloring_files') - self.options['coloring_dir'] = str(default_coloring_dir) - if 'total_coloring' in self._reports: - pathlib.Path(self.options['coloring_dir']).mkdir(parents=True, exist_ok=True) - # this metadata will be shared by all Systems/Solvers in the system tree self._metadata = { 'name': self._name, # the name of this Problem diff --git a/openmdao/core/tests/test_coloring.py b/openmdao/core/tests/test_coloring.py index 1298775afb..0d99d45db5 100644 --- a/openmdao/core/tests/test_coloring.py +++ b/openmdao/core/tests/test_coloring.py @@ -1537,44 +1537,5 @@ def test_wrong_pickle(self): self.assertEqual(ctx.exception.args[0], "File '_bad_pickle_' is not a valid coloring file.") - - -@use_tempdirs -class TestSettingColoringDir(unittest.TestCase): - - def test_coloring_dir_is_None(self): - problem_options = { - 'coloring_dir': None, - } - p = run_opt(pyOptSparseDriver, 'auto', assemble_type='csc', optimizer='SLSQP', - dynamic_total_coloring=True, print_results=False, - problem_options=problem_options) - - coloring_dir = pathlib.Path('./coloring_files') - self.assertTrue(coloring_dir.is_dir(), 'Coloring dir not found') - - def test_coloring_dir_is_custom(self): - problem_options = { - 'coloring_dir': 'custom_coloring_dir', - } - p = run_opt(pyOptSparseDriver, 'auto', assemble_type='csc', optimizer='SLSQP', - dynamic_total_coloring=True, print_results=False, problem_options=problem_options) - - coloring_dir = pathlib.Path('custom_coloring_dir') - self.assertTrue(coloring_dir.is_dir(), 'Coloring dir not found') - - def test_coloring_dir_is_default(self): - problem_options = { - } - p = run_opt(pyOptSparseDriver, 'auto', assemble_type='csc', optimizer='SLSQP', - dynamic_total_coloring=True, print_results=False, - problem_options=problem_options) - - coloring_dir = pathlib.Path(get_reports_dir()).joinpath(p._name).joinpath( - 'coloring_files') - self.assertTrue(coloring_dir.is_dir(), 'Coloring dir not found') - - - if __name__ == '__main__': unittest.main() From dcf0eb163966b812c64bfe4027c5bd7bdc38643b Mon Sep 17 00:00:00 2001 From: Herb Schilling Date: Tue, 21 Nov 2023 10:20:01 -0500 Subject: [PATCH 14/22] Pep8 fixes, removed NLPQLP and ParOpt since no one uses them and the ParOpt does not work, also removed code that shouldn't have been there for handling the user defined location of the output directory for the optimizers --- openmdao/drivers/pyoptsparse_driver.py | 16 ++++------------ .../drivers/tests/test_pyoptsparse_driver.py | 11 ++--------- 2 files changed, 6 insertions(+), 21 deletions(-) diff --git a/openmdao/drivers/pyoptsparse_driver.py b/openmdao/drivers/pyoptsparse_driver.py index 9a8a4832d4..c44d85dfa0 100644 --- a/openmdao/drivers/pyoptsparse_driver.py +++ b/openmdao/drivers/pyoptsparse_driver.py @@ -269,10 +269,10 @@ def _declare_options(self): self.options.declare('hotstart_file', types=str, default=None, allow_none=True, desc='File location of a pyopt_sparse optimization history to use ' 'to hot start the optimization. Default is None.') - self.options.declare('output_dir', types=(str,_ReprClass), default=_DEFAULT_REPORTS_DIR, allow_none=True, + self.options.declare('output_dir', types=(str, _ReprClass), default=_DEFAULT_REPORTS_DIR, + allow_none=True, desc='Directory location of pyopt_sparse output files.' - 'Default is ./reports_directory/problem_name.') - + 'Default is ./reports_directory/problem_name.') @property def hist_file(self): @@ -389,7 +389,7 @@ def run(self): self.options['output_dir'] = str(default_output_dir) output_dir = self.options['output_dir'] - if optimizer == 'ALPSO': # Actually, this is the root of two files generated + if optimizer == 'ALPSO': # Actually, this is the root of two files generated if problem.driver.opt_settings.get('filename') is None: problem.driver.opt_settings['filename'] = f'{output_dir}/ALPSO.out' elif optimizer == 'CONMIN': @@ -398,18 +398,10 @@ def run(self): elif optimizer == 'IPOPT': if problem.driver.opt_settings.get('output_file') is None: problem.driver.opt_settings['output_file'] = f'{output_dir}/IPOPT.out' - elif optimizer == 'NLPQLP': - if problem.driver.opt_settings.get('iFile') is None: - problem.driver.opt_settings['iFile'] = f'{output_dir}/NLPQLP.out' # Nothing for NSGA2 elif optimizer == 'PSQP': if problem.driver.opt_settings.get('IFILE') is None: problem.driver.opt_settings['IFILE'] = f'{output_dir}/PSQP.out' - elif optimizer == 'PAROPT': - if problem.driver.opt_settings.get('tr_output_file') is None: - problem.driver.opt_settings['tr_output_file'] = f'{output_dir}/paropt.tr' - if problem.driver.opt_settings.get('output_file') is None: - problem.driver.opt_settings['output_file'] = f'{output_dir}/paropt.out' elif optimizer == 'SLSQP': if problem.driver.opt_settings.get('IFILE') is None: problem.driver.opt_settings['IFILE'] = f'{output_dir}/SLSQP.out' diff --git a/openmdao/drivers/tests/test_pyoptsparse_driver.py b/openmdao/drivers/tests/test_pyoptsparse_driver.py index b4c6e264f8..0c6634316b 100644 --- a/openmdao/drivers/tests/test_pyoptsparse_driver.py +++ b/openmdao/drivers/tests/test_pyoptsparse_driver.py @@ -3266,9 +3266,7 @@ class TestPyoptSparseOutputFiles(unittest.TestCase): 'ALPSO': [('filename', 'ALPSO_print.out'), ('filename', 'ALPSO_summary.out')], 'CONMIN': [('IFILE', 'CONMIN.out')], 'IPOPT': [('output_file', 'IPOPT.out')], - 'NLPQLP': [('iFile', 'NLPQLP.out')], 'PSQP': [('IFILE', 'PSQP.out')], - 'PAROPT': [('tr_output_file', 'paropt.tr'), ('output_file', 'paropt.out')], 'SLSQP': [('IFILE', 'SLSQP.out')], 'SNOPT': [('Print file', 'SNOPT_print.out'), ('Summary file', 'SNOPT_summary.out')] } @@ -3333,13 +3331,8 @@ def run_and_test_user_set_output_dir(self, optimizer, output_file_names): if optimizer == 'ALPSO': prob.driver.opt_settings['fileout'] = 3 - for opt_setting_name, output_file_name in output_file_names: - if optimizer == 'ALPSO': # Since ALPSO only has the one setting that affects both - # output files - output_file_path = pathlib.Path(user_directory_name).joinpath('ALPSO.out') - else: - output_file_path = pathlib.Path(user_directory_name).joinpath(output_file_name) - prob.driver.opt_settings[opt_setting_name] = str(output_file_path) + + prob.driver.options['output_dir'] = user_directory_name prob.run_driver() From cde00b276357f832c0a1c07d6cb06bdc80124a0f Mon Sep 17 00:00:00 2001 From: Herb Schilling Date: Tue, 21 Nov 2023 10:36:03 -0500 Subject: [PATCH 15/22] Some final minor formatting fixes --- openmdao/core/problem.py | 3 +-- openmdao/utils/tests/test_reports_system.py | 1 - 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/openmdao/core/problem.py b/openmdao/core/problem.py index ae5ab7c51a..7f0295afcf 100644 --- a/openmdao/core/problem.py +++ b/openmdao/core/problem.py @@ -19,8 +19,7 @@ import numpy as np import scipy.sparse as sparse -from openmdao.core.constants import _SetupStatus, _DEFAULT_REPORTS_DIR, _ReprClass - +from openmdao.core.constants import _SetupStatus from openmdao.core.component import Component from openmdao.core.driver import Driver, record_iteration, SaveOptResult from openmdao.core.explicitcomponent import ExplicitComponent diff --git a/openmdao/utils/tests/test_reports_system.py b/openmdao/utils/tests/test_reports_system.py index 433b2586a0..a4501c1085 100644 --- a/openmdao/utils/tests/test_reports_system.py +++ b/openmdao/utils/tests/test_reports_system.py @@ -135,7 +135,6 @@ def solve(self): return super().solve() - prob = om.Problem(reports=prob1_reports) prob.model.add_subsystem('indep', om.IndepVarComp('x', 1.0)) G = prob.model.add_subsystem('G', om.Group()) From 315dd449696ce471d5683611450639f27e993343 Mon Sep 17 00:00:00 2001 From: Herb Schilling Date: Tue, 21 Nov 2023 10:45:08 -0500 Subject: [PATCH 16/22] removed unneeded import --- openmdao/core/tests/test_coloring.py | 1 - 1 file changed, 1 deletion(-) diff --git a/openmdao/core/tests/test_coloring.py b/openmdao/core/tests/test_coloring.py index 9e7bc4ee5d..5479130001 100644 --- a/openmdao/core/tests/test_coloring.py +++ b/openmdao/core/tests/test_coloring.py @@ -1,6 +1,5 @@ import os -import pathlib import sys import itertools import pickle From 767098f5bc54f10bba753411aed059a7ce88c78a Mon Sep 17 00:00:00 2001 From: Herb Schilling Date: Mon, 27 Nov 2023 13:30:32 -0500 Subject: [PATCH 17/22] Need to set output_dir to None so output files go in current directory, which this code expects --- .../advanced_user_guide/analysis_errors/analysis_error.ipynb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/openmdao/docs/openmdao_book/advanced_user_guide/analysis_errors/analysis_error.ipynb b/openmdao/docs/openmdao_book/advanced_user_guide/analysis_errors/analysis_error.ipynb index 9ab5c23f1d..d143b1b1ab 100644 --- a/openmdao/docs/openmdao_book/advanced_user_guide/analysis_errors/analysis_error.ipynb +++ b/openmdao/docs/openmdao_book/advanced_user_guide/analysis_errors/analysis_error.ipynb @@ -102,6 +102,7 @@ " if optimizer == 'IPOPT':\n", " driver.opt_settings['file_print_level'] = 5\n", " driver.options['print_results'] = False\n", + " driver.options['output_dir'] = None # will put the optimizer output file in the current directory\n", "\n", " # setup problem & initialize values\n", " prob = om.Problem(model, driver)\n", @@ -424,7 +425,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.10.6" + "version": "3.11.6" } }, "nbformat": 4, From 7587d0bfa58752886d45093e96162b0b6a3f0da2 Mon Sep 17 00:00:00 2001 From: Herb Schilling Date: Mon, 27 Nov 2023 13:36:27 -0500 Subject: [PATCH 18/22] removed coloring test code for changing coloring files location. Decided to postpone this --- openmdao/core/tests/test_coloring.py | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/openmdao/core/tests/test_coloring.py b/openmdao/core/tests/test_coloring.py index 5479130001..db720fe641 100644 --- a/openmdao/core/tests/test_coloring.py +++ b/openmdao/core/tests/test_coloring.py @@ -10,9 +10,6 @@ from io import StringIO from numpy.testing import assert_almost_equal - -from openmdao.utils.reports_system import get_reports_dir - try: from scipy.sparse import load_npz except ImportError: @@ -92,9 +89,7 @@ def compute(self, inputs, outputs): def run_opt(driver_class, mode, assemble_type=None, color_info=None, derivs=True, recorder=None, has_lin_constraint=True, has_diag_partials=True, partial_coloring=False, - use_vois=True, auto_ivc=False, con_alias=False, check=False, - problem_options=None, - **options): + use_vois=True, auto_ivc=False, con_alias=False, check=False, **options): p = om.Problem(model=CounterGroup()) @@ -257,9 +252,6 @@ def compute(self, inputs, outputs): if recorder: p.driver.add_recorder(recorder) - if problem_options and 'coloring_dir' in problem_options: - p.options['coloring_dir'] = problem_options['coloring_dir'] - p.setup(mode=mode, derivatives=derivs, check=check) if use_vois: p.run_driver() @@ -1562,5 +1554,6 @@ def test_wrong_pickle(self): self.assertEqual(ctx.exception.args[0], "File '_bad_pickle_' is not a valid coloring file.") + if __name__ == '__main__': unittest.main() From 0580e421fad52f9d9a827b137ac286293d33ea78 Mon Sep 17 00:00:00 2001 From: Herb Schilling Date: Tue, 28 Nov 2023 14:11:06 -0500 Subject: [PATCH 19/22] used the wrong string for the ReprClass --- openmdao/core/constants.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openmdao/core/constants.py b/openmdao/core/constants.py index cea5f833a5..5693bacb93 100644 --- a/openmdao/core/constants.py +++ b/openmdao/core/constants.py @@ -135,4 +135,4 @@ def __deepcopy__(self, memo): # does not have a reference to the Problem so can't get the name. This serves as a flag that # the default directory in the reports directory is what is wanted. Then in the run method, # the actual default directory is used -_DEFAULT_REPORTS_DIR = _ReprClass("DEFAULT_PYOPT_SPARSE_OUTPUT_DIR") +_DEFAULT_REPORTS_DIR = _ReprClass("DEFAULT_REPORTS_DIR") From ae0d3e5fd114807b26ce34f60256fa4001fa03ce Mon Sep 17 00:00:00 2001 From: Herb Schilling Date: Mon, 4 Dec 2023 14:01:26 -0500 Subject: [PATCH 20/22] Updates based on reviewer comments on how to better write code for output_dir --- openmdao/drivers/pyoptsparse_driver.py | 58 ++++++++++++-------------- 1 file changed, 26 insertions(+), 32 deletions(-) diff --git a/openmdao/drivers/pyoptsparse_driver.py b/openmdao/drivers/pyoptsparse_driver.py index 73a5bf086e..c8acbc7a40 100644 --- a/openmdao/drivers/pyoptsparse_driver.py +++ b/openmdao/drivers/pyoptsparse_driver.py @@ -380,38 +380,6 @@ def run(self): optimizer = self.options['optimizer'] - # Need to tell optimizer where to put its .out files - if self.options['output_dir'] is None: - self.options['output_dir'] = "." - elif self.options['output_dir'] == _DEFAULT_REPORTS_DIR: - problem = self._problem() - default_output_dir = pathlib.Path(get_reports_dir()).joinpath(problem._name) - pathlib.Path(default_output_dir).mkdir(parents=True, exist_ok=True) - self.options['output_dir'] = str(default_output_dir) - output_dir = self.options['output_dir'] - - if optimizer == 'ALPSO': # Actually, this is the root of two files generated - if problem.driver.opt_settings.get('filename') is None: - problem.driver.opt_settings['filename'] = f'{output_dir}/ALPSO.out' - elif optimizer == 'CONMIN': - if problem.driver.opt_settings.get('IFILE') is None: - problem.driver.opt_settings['IFILE'] = f'{output_dir}/CONMIN.out' - elif optimizer == 'IPOPT': - if problem.driver.opt_settings.get('output_file') is None: - problem.driver.opt_settings['output_file'] = f'{output_dir}/IPOPT.out' - # Nothing for NSGA2 - elif optimizer == 'PSQP': - if problem.driver.opt_settings.get('IFILE') is None: - problem.driver.opt_settings['IFILE'] = f'{output_dir}/PSQP.out' - elif optimizer == 'SLSQP': - if problem.driver.opt_settings.get('IFILE') is None: - problem.driver.opt_settings['IFILE'] = f'{output_dir}/SLSQP.out' - elif optimizer == 'SNOPT': - if problem.driver.opt_settings.get('Print file') is None: - problem.driver.opt_settings['Print file'] = f'{output_dir}/SNOPT_print.out' - if problem.driver.opt_settings.get('Summary file') is None: - problem.driver.opt_settings['Summary file'] = f'{output_dir}/SNOPT_summary.out' - self._fill_NANs = not respects_fail_flag[self.options['optimizer']] self._check_for_missing_objective() @@ -602,6 +570,32 @@ def run(self): msg = "Optimizer %s is not available in this installation." % optimizer raise ImportError(msg) + # Need to tell optimizer where to put its .out files + if self.options['output_dir'] is None: + output_dir = "." + elif self.options['output_dir'] == _DEFAULT_REPORTS_DIR: + problem = self._problem() + default_output_dir = pathlib.Path(get_reports_dir()).joinpath(problem._name) + pathlib.Path(default_output_dir).mkdir(parents=True, exist_ok=True) + output_dir = str(default_output_dir) + else: + output_dir = self.options['output_dir'] + + + optimizers_and_output_files = { + # ALPSO uses a single option `filename` to determine name of both output files + 'ALPSO': [('filename', 'ALPSO.out')], + 'CONMIN': [('IFILE', 'CONMIN.out')], + 'IPOPT': [('output_file', 'IPOPT.out')], + 'PSQP': [('IFILE', 'PSQP.out')], + 'SLSQP': [('IFILE', 'SLSQP.out')], + 'SNOPT': [('Print file', 'SNOPT_print.out'), ('Summary file', 'SNOPT_summary.out')] + } + + for opt_setting_name, output_file_name in optimizers_and_output_files[optimizer]: + if self.opt_settings.get(opt_setting_name) is None: + self.opt_settings[opt_setting_name] = f'{output_dir}/{output_file_name}' + # Process any default optimizer-specific settings. if optimizer in DEFAULT_OPT_SETTINGS: for name, value in DEFAULT_OPT_SETTINGS[optimizer].items(): From 1a0027412517c90ef9a093d113b00f9f6d384250 Mon Sep 17 00:00:00 2001 From: Herb Schilling Date: Mon, 4 Dec 2023 14:11:18 -0500 Subject: [PATCH 21/22] Fixed PEP8 errors --- openmdao/drivers/pyoptsparse_driver.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/openmdao/drivers/pyoptsparse_driver.py b/openmdao/drivers/pyoptsparse_driver.py index c8acbc7a40..e57fcb9e45 100644 --- a/openmdao/drivers/pyoptsparse_driver.py +++ b/openmdao/drivers/pyoptsparse_driver.py @@ -581,7 +581,6 @@ def run(self): else: output_dir = self.options['output_dir'] - optimizers_and_output_files = { # ALPSO uses a single option `filename` to determine name of both output files 'ALPSO': [('filename', 'ALPSO.out')], @@ -595,7 +594,7 @@ def run(self): for opt_setting_name, output_file_name in optimizers_and_output_files[optimizer]: if self.opt_settings.get(opt_setting_name) is None: self.opt_settings[opt_setting_name] = f'{output_dir}/{output_file_name}' - + # Process any default optimizer-specific settings. if optimizer in DEFAULT_OPT_SETTINGS: for name, value in DEFAULT_OPT_SETTINGS[optimizer].items(): From 0349fcf6b640440456c72ac86c19f7674cce2053 Mon Sep 17 00:00:00 2001 From: Herb Schilling Date: Mon, 4 Dec 2023 15:57:11 -0500 Subject: [PATCH 22/22] Handle optimizer that does not have output files --- openmdao/drivers/pyoptsparse_driver.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/openmdao/drivers/pyoptsparse_driver.py b/openmdao/drivers/pyoptsparse_driver.py index e57fcb9e45..0f87c1f978 100644 --- a/openmdao/drivers/pyoptsparse_driver.py +++ b/openmdao/drivers/pyoptsparse_driver.py @@ -591,9 +591,10 @@ def run(self): 'SNOPT': [('Print file', 'SNOPT_print.out'), ('Summary file', 'SNOPT_summary.out')] } - for opt_setting_name, output_file_name in optimizers_and_output_files[optimizer]: - if self.opt_settings.get(opt_setting_name) is None: - self.opt_settings[opt_setting_name] = f'{output_dir}/{output_file_name}' + if optimizer in optimizers_and_output_files: + for opt_setting_name, output_file_name in optimizers_and_output_files[optimizer]: + if self.opt_settings.get(opt_setting_name) is None: + self.opt_settings[opt_setting_name] = f'{output_dir}/{output_file_name}' # Process any default optimizer-specific settings. if optimizer in DEFAULT_OPT_SETTINGS: