diff --git a/openmdao/core/group.py b/openmdao/core/group.py index 672b0dc739..107b2631a8 100644 --- a/openmdao/core/group.py +++ b/openmdao/core/group.py @@ -5099,3 +5099,39 @@ def _gather_full_data(self): self._subsystems_myproc[0]._full_comm.rank == 0 return False + + def _get_prom_name(self, abs_name): + """ + Get promoted name for specified variable. + """ + abs2prom = self._var_allprocs_abs2prom + if abs_name in abs2prom['input']: + return abs2prom['input'][abs_name] + elif abs_name in abs2prom['output']: + return abs2prom['output'][abs_name] + else: + return abs_name + + def _prom_names_list(self, lst): + """ + Convert a list of variable names to promoted names. + """ + return [self._get_prom_name(n) for n in lst] + + def _prom_names_dict(self, dct): + """ + Convert a dictionary keyed on variable names to be keyed on promoted names. + """ + return {self._get_prom_name(k): v for k, v in dct.items()} + + def _prom_names_jac(self, jac): + """ + Convert a nested dict jacobian keyed on variable names to be keyed on promoted names. + """ + new_jac = {} + for of in jac: + new_jac[self._get_prom_name(of)] = of_dict = {} + for wrt in jac[of]: + of_dict[self._get_prom_name(wrt)] = jac[of][wrt] + + return new_jac diff --git a/openmdao/core/tests/test_coloring.py b/openmdao/core/tests/test_coloring.py index 0538732a5a..db720fe641 100644 --- a/openmdao/core/tests/test_coloring.py +++ b/openmdao/core/tests/test_coloring.py @@ -125,7 +125,7 @@ def run_opt(driver_class, mode, assemble_type=None, color_info=None, derivs=True p.model.add_subsystem('arctan_yox', arctan_yox) - p.model.add_subsystem('circle', om.ExecComp('area=pi*r**2')) + p.model.add_subsystem('circle', om.ExecComp('area=pi*r**2'), promotes_outputs=['area']) p.model.add_subsystem('r_con', om.ExecComp('g=x**2 + y**2 - r', has_diag_partials=has_diag_partials, g=np.ones(SIZE), x=np.ones(SIZE), y=np.ones(SIZE))) @@ -243,7 +243,7 @@ def compute(self, inputs, outputs): # linear constraint (if has_lin_constraint is set) p.model.add_constraint('y', equals=0, indices=[0,], linear=has_lin_constraint) - p.model.add_objective('circle.area', ref=-1) + p.model.add_objective('area', ref=-1) # setup coloring if color_info is not None: @@ -312,6 +312,32 @@ def test_dynamic_total_coloring_snopt_auto_autoivc(self): self.assertEqual(p.model._solve_count, 21) self.assertEqual(p_color.model._solve_count, 5) + @unittest.skipUnless(OPTIMIZER == 'SNOPT', "This test requires SNOPT.") + def test_dynamic_total_coloring_display_txt(self): + p_color = run_opt(pyOptSparseDriver, 'auto', optimizer='SNOPT', print_results=False, + dynamic_total_coloring=True, auto_ivc=True) + + # check text visualizion of coloring + coloring = compute_total_coloring(p_color) + + stream = StringIO() + coloring.display_txt(out_stream=stream, use_prom_names=False) + color_txt = stream.getvalue().split('\n') + + self.assertTrue(color_txt[0].endswith(' circle.area')) + self.assertEqual(color_txt[22].strip(), '|_auto_ivc.v0') + self.assertEqual(color_txt[23].strip(), '|_auto_ivc.v1') + self.assertEqual(color_txt[24].strip(), '|_auto_ivc.v2') + + stream = StringIO() + coloring.display_txt(out_stream=stream) # use_prom_names=True is the default + color_txt = stream.getvalue().split('\n') + + self.assertTrue(color_txt[0].endswith(' area')) # promoted name + self.assertEqual(color_txt[22].strip(), '|x') # connected input rather than _auto_ivc.v0 + self.assertEqual(color_txt[23].strip(), '|y') # connected input rather than _auto_ivc.v1 + self.assertEqual(color_txt[24].strip(), '|r') # connected input rather than _auto_ivc.v2 + @unittest.skipUnless(OPTIMIZER == 'SNOPT', "This test requires SNOPT.") def test_dynamic_total_coloring_snopt_auto_dyn_partials(self): # first, run w/o coloring @@ -540,7 +566,7 @@ def test_dynamic_total_coloring_pyoptsparse_slsqp_auto(self): # test __repr__ rep = repr(p_color.driver._coloring_info['coloring']) - self.assertEqual(rep.replace('L', ''), 'Coloring (direction: fwd, ncolors: 5, shape: (22, 21), pct nonzero: 13.42, tol: 1e-15') + self.assertEqual(rep.replace('L', ''), 'Coloring (direction: fwd, ncolors: 5, shape: (22, 21), pct nonzero: 13.42, tol: 1e-15)') @unittest.skipUnless(OPTIMIZER == 'SNOPT', "This test requires SNOPT.") def test_print_options_total_with_coloring_fwd(self): @@ -821,7 +847,7 @@ def test_problem_total_coloring_auto(self): p = run_opt(om.ScipyOptimizeDriver, 'auto', optimizer='SLSQP', disp=False, use_vois=False) coloring = compute_total_coloring(p, of=['r_con.g', 'theta_con.g', 'delta_theta_con.g', - 'l_conx.g', 'y', 'circle.area'], + 'l_conx.g', 'y', 'area'], wrt=['x', 'y', 'r']) self.assertEqual(coloring.total_solves(), 5) @@ -1068,12 +1094,12 @@ def test_repr(self): p_color = run_opt(om.ScipyOptimizeDriver, 'auto', optimizer='SLSQP', disp=False, dynamic_total_coloring=True) coloring = p_color.driver._coloring_info['coloring'] rep = repr(coloring) - self.assertEqual(rep.replace('L', ''), 'Coloring (direction: fwd, ncolors: 5, shape: (22, 21), pct nonzero: 13.42, tol: 1e-15') + self.assertEqual(rep.replace('L', ''), 'Coloring (direction: fwd, ncolors: 5, shape: (22, 21), pct nonzero: 13.42, tol: 1e-15)') dense_J = np.ones((50, 50), dtype=bool) coloring = _compute_coloring(dense_J, 'auto') rep = repr(coloring) - self.assertEqual(rep.replace('L', ''), 'Coloring (direction: fwd, ncolors: 50, shape: (50, 50), pct nonzero: 100.00, tol: None') + self.assertEqual(rep.replace('L', ''), 'Coloring (direction: fwd, ncolors: 50, shape: (50, 50), pct nonzero: 100.00, tol: None)') def test_bad_mode(self): p_color_rev = run_opt(om.ScipyOptimizeDriver, 'rev', optimizer='SLSQP', disp=False, dynamic_total_coloring=True) diff --git a/openmdao/core/tests/test_prob_remote.py b/openmdao/core/tests/test_prob_remote.py index 36f7c0b400..39852a7b8a 100644 --- a/openmdao/core/tests/test_prob_remote.py +++ b/openmdao/core/tests/test_prob_remote.py @@ -81,21 +81,29 @@ def test_is_local(self): par.add_subsystem('C2', om.ExecComp('y=3*x')) p.model.connect('indep.x', ['par.C1.x', 'par.C2.x']) - with self.assertRaisesRegex(RuntimeError, - "Problem .*: is_local\('indep\.x'\) was called before setup\(\) completed\."): - loc = p.is_local('indep.x') - - with self.assertRaisesRegex(RuntimeError, - "Problem .*: is_local\('par\.C1'\) was called before setup\(\) completed\."): - loc = p.is_local('par.C1') - - with self.assertRaisesRegex(RuntimeError, - "Problem .*: is_local\('par\.C1\.y'\) was called before setup\(\) completed\."): - loc = p.is_local('par.C1.y') - - with self.assertRaisesRegex(RuntimeError, - "Problem .*: is_local\('par\.C1\.x'\) was called before setup\(\) completed\."): - loc = p.is_local('par.C1.x') + with self.assertRaises(RuntimeError) as err: + p.is_local('indep.x') + self.assertEqual(str(err.exception), + f"Problem {p._get_inst_id()}: is_local('indep.x') " + "was called before setup() completed.") + + with self.assertRaises(RuntimeError) as err: + p.is_local('par.C1') + self.assertEqual(str(err.exception), + f"Problem {p._get_inst_id()}: is_local('par.C1') " + "was called before setup() completed.") + + with self.assertRaises(RuntimeError) as err: + p.is_local('par.C1.y') + self.assertEqual(str(err.exception), + f"Problem {p._get_inst_id()}: is_local('par.C1.y') " + "was called before setup() completed.") + + with self.assertRaises(RuntimeError) as err: + p.is_local('par.C1.x') + self.assertEqual(str(err.exception), + f"Problem {p._get_inst_id()}: is_local('par.C1.x') " + "was called before setup() completed.") p.setup() p.final_setup() diff --git a/openmdao/core/tests/test_problem.py b/openmdao/core/tests/test_problem.py index 8ea016766e..c223327b37 100644 --- a/openmdao/core/tests/test_problem.py +++ b/openmdao/core/tests/test_problem.py @@ -739,17 +739,23 @@ def test_set_cs_error_messages(self): prob.model.add_subsystem('comp', Paraboloid()) prob.setup() prob.run_model() - msg = "Problem .*: To enable complex step, specify 'force_alloc_complex=True' when calling " + \ - "setup on the problem, e\.g\. 'problem\.setup\(force_alloc_complex=True\)'" - with self.assertRaisesRegex(RuntimeError, msg): + + msg = f"Problem {prob._get_inst_id()}: To enable complex step, specify 'force_alloc_complex=True' when calling " + \ + "setup on the problem, e.g. 'problem.setup(force_alloc_complex=True)'" + + with self.assertRaises(RuntimeError) as err: prob.set_complex_step_mode(True) + self.assertEqual(str(err.exception), msg) prob = om.Problem() prob.model.add_subsystem('comp', Paraboloid()) - msg = "Problem .*: set_complex_step_mode cannot be called before `Problem\.run_model\(\)`, " + \ - "`Problem\.run_driver\(\)`, or `Problem\.final_setup\(\)`." - with self.assertRaisesRegex(RuntimeError, msg) : + + msg = f"Problem {prob._get_inst_id()}: set_complex_step_mode cannot be called before `Problem.run_model()`, " + \ + "`Problem.run_driver()`, or `Problem.final_setup()`." + + with self.assertRaises(RuntimeError) as err: prob.set_complex_step_mode(True) + self.assertEqual(str(err.exception), msg) def test_feature_run_driver(self): @@ -1818,19 +1824,23 @@ def test_list_problem_vars_before_final_setup(self): prob.model.add_subsystem('const', om.ExecComp('g = x + y'), promotes_inputs=['x', 'y']) prob.model.set_input_defaults('x', 3.0) prob.model.set_input_defaults('y', -4.0) + prob.driver = om.ScipyOptimizeDriver() prob.driver.options['optimizer'] = 'COBYLA' + prob.model.add_design_var('x', lower=-50, upper=50) prob.model.add_design_var('y', lower=-50, upper=50) prob.model.add_objective('parab.f_xy') prob.model.add_constraint('const.g', lower=0, upper=10.) + prob.setup() - msg = "Problem .*: Problem.list_problem_vars\(\) cannot be called before " \ - "`Problem\.run_model\(\)`, `Problem\.run_driver\(\)`, or " \ - "`Problem\.final_setup\(\)`\." - with self.assertRaisesRegex(RuntimeError, msg): + msg = f"Problem {prob._get_inst_id()}: Problem.list_problem_vars() cannot be called " \ + "before `Problem.run_model()`, `Problem.run_driver()`, or `Problem.final_setup()`." + + with self.assertRaises(RuntimeError) as err: prob.list_problem_vars() + self.assertEqual(str(err.exception), msg) def test_list_problem_w_multi_constraints(self): p = om.Problem() diff --git a/openmdao/docs/openmdao_book/features/core_features/working_with_derivatives/simul_derivs.ipynb b/openmdao/docs/openmdao_book/features/core_features/working_with_derivatives/simul_derivs.ipynb index 23ede75bc0..244b4d0e76 100644 --- a/openmdao/docs/openmdao_book/features/core_features/working_with_derivatives/simul_derivs.ipynb +++ b/openmdao/docs/openmdao_book/features/core_features/working_with_derivatives/simul_derivs.ipynb @@ -34,7 +34,7 @@ "\n", "When mode is set to ‘fwd’ or ‘rev’, a unidirectional coloring algorithm is used to group columns or rows, respectively, for simultaneous derivative calculation. The algorithm used in this case is the greedy algorithm with ordering by incidence degree found in T. F. Coleman and J. J. More, *Estimation of sparse Jacobian matrices and graph coloring problems*, SIAM J. Numer. Anal., 20 (1983), pp. 187–209.\n", "\n", - "When using simultaneous derivatives, setting mode=’auto’ will indicate that bidirectional coloring should be used. Bidirectional coloring can significantly decrease the number of linear solves needed to generate the total Jacobian relative to coloring only in fwd or rev mode.\n", + "When using simultaneous derivatives, setting `mode=’auto’` will indicate that bidirectional coloring should be used. Bidirectional coloring can significantly decrease the number of linear solves needed to generate the total Jacobian relative to coloring only in fwd or rev mode.\n", "\n", "For more information on the bidirectional coloring algorithm, see T. F. Coleman and A. Verma, *The efficient computation of sparse Jacobian matrices using automatic differentiation*, SIAM J. Sci. Comput., 19 (1998), pp. 1210–1233." ] @@ -248,14 +248,17 @@ "id": "954218f7", "metadata": {}, "source": [ - "(view_coloring-api)=\n", - "## openmdao.api.view_coloring\n", + "(display_coloring-api)=\n", + "## Viewing total coloring from a script\n", "\n", - "The `openmdao.api` includes a view_coloring command to allow viewing of coloring data from within a script.\n", - "This command will generate an html file, but will render the coloring as plain text if bokeh is unavailable or if the `as_text` argument is True.\n", + "The `openmdao.api` includes a `display_coloring` function to allow viewing of coloring data from within a script.\n", + "This function will generate an html file, but will render the coloring as plain text if bokeh is unavailable or if the `as_text` argument is True.\n", "\n", - ".. autofunction:: openmdao.utils.coloring.view_coloring\n", - " :noindex:" + "```{eval-rst}\n", + " .. autofunction:: openmdao.utils.coloring.display_coloring\n", + " :noindex:\n", + "```\n", + "\n" ] }, { diff --git a/openmdao/docs/openmdao_book/theory_manual/advanced_linear_solvers_special_cases/separable.ipynb b/openmdao/docs/openmdao_book/theory_manual/advanced_linear_solvers_special_cases/separable.ipynb index 22915cab95..3ac6b41d45 100644 --- a/openmdao/docs/openmdao_book/theory_manual/advanced_linear_solvers_special_cases/separable.ipynb +++ b/openmdao/docs/openmdao_book/theory_manual/advanced_linear_solvers_special_cases/separable.ipynb @@ -90,7 +90,7 @@ "Hence, it would be faster to solve for total derivatives using forward mode with simultaneous\n", "derivatives than reverse mode.\n", "\n", - "Determining if Your Problem is Separable\n", + "## Determining if Your Problem is Separable\n", "\n", "The simple example above was contrived to make it relatively obvious that the problem was separable.\n", "For realistic problems, even if you know that the problem should be separable, computing the actual\n", @@ -123,7 +123,8 @@ "If the model is intended to be used in an optimization context, then it is fair to assume that the total-derivative Jacobian is inexpensive enough to compute many times,\n", "and using a few additional computations to compute a coloring will not significantly impact the overall compute cost.\n", "\n", - "Choosing Forward or Reverse Mode for Separable Problems\n", + "## Choosing Forward or Reverse Mode for Separable Problems\n", + "\n", "If a problem has a section of design variables and constraints that are separable,\n", "then it is possible to leverage that quality in either forward or reverse mode.\n", "Which mode you choose depends on which direction gives you fewer total linear solves.\n", @@ -134,12 +135,13 @@ "Sometimes the answer is different than you would get by counting design variables and constraints, but sometimes its not.\n", "The result is problem-dependent.\n", "\n", - "Relevance to Finite Difference and Complex Step\n", + "## Relevance to Finite Difference and Complex Step\n", + "\n", "It is worth noting that, in addition to speeding up linear solutions for the unified derivative equations, forward separability also offers benefits when finite difference or complex step are being used to compute derivatives numerically.\n", "For the same reasons that multiple linear solves can be combined, you can also take steps in multiple variables to compute derivatives with respect to multiple variables at the same time.\n", "\n", + "## How to actually use it!\n", "\n", - "How to actually use it!\n", "OpenMDAO provides a mechanism for you to specify a coloring to take advantage of separability, via the\n", "[use_fixed_coloring](../../features/core_features/working_with_derivatives/simul_derivs.ipynb) method.\n", "OpenMDAO also provides a [coloring tool](static-coloring) to determine the minimum number of colors your problem can be reduced to.\n", diff --git a/openmdao/drivers/doe_driver.py b/openmdao/drivers/doe_driver.py index 1b6b19a2d4..b7fb20d23f 100644 --- a/openmdao/drivers/doe_driver.py +++ b/openmdao/drivers/doe_driver.py @@ -34,9 +34,9 @@ class DOEDriver(Driver): _color : int or None In MPI, the cached color is used to determine which cases to run on this proc. _indep_list : list - List of design variables. + List of design variables, used to compute derivatives. _quantities : list - Contains the objectives plus nonlinear constraints. + Contains the objectives plus nonlinear constraints, used to compute derivatives. """ def __init__(self, generator=None, **kwargs): @@ -232,8 +232,7 @@ def _run_case(self, case): # save reference to metadata for use in record_iteration self._metadata = metadata - opts = self.recording_options - if opts['record_derivatives']: + if self.recording_options['record_derivatives']: self._compute_totals(of=self._quantities, wrt=self._indep_list, return_format=self._total_jac_format, diff --git a/openmdao/drivers/pyoptsparse_driver.py b/openmdao/drivers/pyoptsparse_driver.py index 4219e349fe..c983a97111 100644 --- a/openmdao/drivers/pyoptsparse_driver.py +++ b/openmdao/drivers/pyoptsparse_driver.py @@ -366,6 +366,7 @@ def run(self): problem = self._problem() model = problem.model relevant = model._relevant + self.pyopt_solution = None self._total_jac = None self.iter_count = 0 @@ -400,16 +401,22 @@ def run(self): # Add all design variables self._indep_list = indep_list = list(self._designvars) + self._indep_list_prom = indep_list_prom = [] + input_vals = self.get_design_var_values() for name, meta in self._designvars.items(): + # translate absolute var names to promoted names for pyoptsparse + prom_name = model._get_prom_name(name) + indep_list_prom.append(prom_name) + size = meta['global_size'] if meta['distributed'] else meta['size'] if pyoptsparse_version is None or pyoptsparse_version < Version('2.6.1'): - opt_prob.addVarGroup(name, size, type='c', + opt_prob.addVarGroup(prom_name, size, type='c', value=input_vals[name], lower=meta['lower'], upper=meta['upper']) else: - opt_prob.addVarGroup(name, size, varType='c', + opt_prob.addVarGroup(prom_name, size, varType='c', value=input_vals[name], lower=meta['lower'], upper=meta['upper']) @@ -421,7 +428,7 @@ def run(self): # Add all objectives objs = self.get_objective_values() for name in objs: - opt_prob.addObj(name) + opt_prob.addObj(model._get_prom_name(name)) self._quantities.append(name) cons_to_remove = set() @@ -463,26 +470,35 @@ def run(self): rels = relevant[path] wrt = [v for v in indep_list if self._designvars[v]['source'] in rels] + prom_name = model._get_prom_name(name) + if not wrt: - issue_warning(f"Equality constraint '{name}' does not depend on any design " + issue_warning(f"Equality constraint '{prom_name}' does not depend on any design " "variables and was not added to the optimization.") cons_to_remove.add(name) continue + # convert wrt to use promoted names + wrt_prom = model._prom_names_list(wrt) + if meta['linear']: jac = {w: _lin_jacs[name][w] for w in wrt} - opt_prob.addConGroup(name, size, + jac_prom = model._prom_names_dict(jac) + opt_prob.addConGroup(prom_name, size, lower=lower - _y_intercepts[name], upper=upper - _y_intercepts[name], - linear=True, wrt=wrt, jac=jac) + linear=True, wrt=wrt_prom, jac=jac_prom) else: if name in self._res_subjacs: resjac = self._res_subjacs[name] jac = {n: resjac[self._designvars[n]['source']] for n in wrt} + jac_prom = model._prom_names_jac(jac) else: jac = None + jac_prom = None - opt_prob.addConGroup(name, size, lower=lower, upper=upper, wrt=wrt, jac=jac) + opt_prob.addConGroup(prom_name, size, lower=lower, upper=upper, + wrt=wrt_prom, jac=jac_prom) self._quantities.append(name) # Add all inequality constraints @@ -503,25 +519,34 @@ def run(self): rels = relevant[path] wrt = [v for v in indep_list if self._designvars[v]['source'] in rels] + prom_name = model._get_prom_name(name) + if not wrt: - issue_warning(f"Inequality constraint '{name}' does not depend on any design " + issue_warning(f"Inequality constraint '{prom_name}' does not depend on any design " "variables and was not added to the optimization.") cons_to_remove.add(name) continue + # convert wrt to use promoted names + wrt_prom = model._prom_names_list(wrt) + if meta['linear']: jac = {w: _lin_jacs[name][w] for w in wrt} - opt_prob.addConGroup(name, size, + jac_prom = model._prom_names_dict(jac) + opt_prob.addConGroup(prom_name, size, upper=upper - _y_intercepts[name], lower=lower - _y_intercepts[name], - linear=True, wrt=wrt, jac=jac) + linear=True, wrt=wrt_prom, jac=jac_prom) else: if name in self._res_subjacs: resjac = self._res_subjacs[name] jac = {n: resjac[self._designvars[n]['source']] for n in wrt} + jac_prom = model._prom_names_jac(jac) else: jac = None - opt_prob.addConGroup(name, size, upper=upper, lower=lower, wrt=wrt, jac=jac) + jac_prom = None + opt_prob.addConGroup(prom_name, size, upper=upper, lower=lower, + wrt=wrt_prom, jac=jac_prom) self._quantities.append(name) for name in cons_to_remove: @@ -610,7 +635,7 @@ def run(self): # framework is left in the right final state dv_dict = sol.getDVs() for name in indep_list: - self.set_design_var(name, dv_dict[name]) + self.set_design_var(name, dv_dict[model._get_prom_name(name)]) with RecordingDebugging(self._get_name(), self.iter_count, self) as rec: try: @@ -681,7 +706,7 @@ def _objfunc(self, dv_dict): try: for name in self._indep_list: - self.set_design_var(name, dv_dict[name]) + self.set_design_var(name, dv_dict[model._get_prom_name(name)]) # print("Setting DV") # print(dv_dict) @@ -690,6 +715,8 @@ def _objfunc(self, dv_dict): if self._user_termination_flag: func_dict = self.get_objective_values() func_dict.update(self.get_constraint_values(lintype='nonlinear')) + # convert func_dict to use promoted names + func_dict = model._prom_names_dict(func_dict) return func_dict, 2 # Execute the model @@ -726,6 +753,9 @@ def _objfunc(self, dv_dict): for name in func_dict: func_dict[name].fill(np.NAN) + # convert func_dict to use promoted names + func_dict = model._prom_names_dict(func_dict) + # print("Functions calculated") # print(dv_dict) # print(func_dict, flush=True) @@ -758,6 +788,7 @@ def _gradfunc(self, dv_dict, func_dict): 1 for unsuccessful function evaluation """ prob = self._problem() + model = prob.model fail = 0 sens_dict = {} @@ -796,9 +827,10 @@ def _gradfunc(self, dv_dict, func_dict): # TODO: look into getting rid of all of these conversions! new_sens = {} res_subjacs = self._res_subjacs - for okey in func_dict: + + for okey in self._quantities: new_sens[okey] = newdv = {} - for ikey in dv_dict: + for ikey in self._designvars.keys(): ikey_src = self._designvars[ikey]['source'] if okey in res_subjacs and ikey_src in res_subjacs[okey]: arr = sens_dict[okey][ikey] @@ -818,17 +850,22 @@ def _gradfunc(self, dv_dict, func_dict): if fail > 0: # We need to cobble together a sens_dict of the correct size. # Best we can do is return zeros or NaNs. - for okey, oval in func_dict.items(): + for okey in self._quantities: if okey not in sens_dict: sens_dict[okey] = {} + oval = func_dict[model._get_prom_name(okey)] osize = len(oval) - for ikey, ival in dv_dict.items(): + for ikey in self._designvars.keys(): + ival = dv_dict[model._get_prom_name(ikey)] isize = len(ival) if ikey not in sens_dict[okey] or self._fill_NANs: sens_dict[okey][ikey] = np.zeros((osize, isize)) if self._fill_NANs: sens_dict[okey][ikey].fill(np.NAN) + # convert sens_dict to use promoted names + sens_dict = model._prom_names_jac(sens_dict) + # print("Derivatives calculated") # print(dv_dict) # print(sens_dict, flush=True) diff --git a/openmdao/drivers/tests/test_doe_driver.py b/openmdao/drivers/tests/test_doe_driver.py index 905d967455..36ead070c9 100644 --- a/openmdao/drivers/tests/test_doe_driver.py +++ b/openmdao/drivers/tests/test_doe_driver.py @@ -120,7 +120,7 @@ def test_no_pyDOE3(self): " pip install openmdao[doe]\n" " pip install pyDOE3") - @unittest.skipUnless(pyDOE3, "requires 'pyDOE3', install openmdao[doe]") + @unittest.skipUnless(pyDOE3, "requires 'pyDOE3', pip install openmdao[doe]") def test_lhc_criterion(self): with self.assertRaises(ValueError) as err: om.LatinHypercubeGenerator(criterion='foo') @@ -348,7 +348,7 @@ def test_csv(self): for name in ('x', 'y', 'f_xy'): self.assertEqual(outputs[name], expected_case[name]) - @unittest.skipUnless(pyDOE3, "requires 'pyDOE3', install openmdao[doe]") + @unittest.skipUnless(pyDOE3, "requires 'pyDOE3', pip install openmdao[doe]") def test_csv_array(self): prob = om.Problem() model = prob.model @@ -484,18 +484,11 @@ def test_csv_errors(self): for case in cases: writer.writerow([np.ones((2, 2)) * val for _, val in case]) - from packaging.version import Version - if Version(np.__version__) >= Version("1.14"): - opts = {'legacy': '1.13'} - else: - opts = {} - - with printoptions(**opts): - # have to use regex to handle differences in numpy print formats for shape - msg = f"Error assigning p1.x = \[ 0. 0. 0. 0.\]: could not broadcast " \ - f"input array from shape \(4.*\) into shape \(1.*\)" - with self.assertRaisesRegex(ValueError, msg): - prob.run_driver() + with self.assertRaises(ValueError) as err: + prob.run_driver() + self.assertEqual(str(err.exception), + "Error assigning p1.x = [0. 0. 0. 0.]: could not broadcast " + "input array from shape (4,) into shape (1,)") def test_uniform(self): prob = om.Problem() @@ -535,7 +528,7 @@ def test_uniform(self): for name in ('x', 'y'): assert_near_equal(outputs[name], expected_case[name], 1e-4) - @unittest.skipUnless(pyDOE3, "requires 'pyDOE3', install openmdao[doe]") + @unittest.skipUnless(pyDOE3, "requires 'pyDOE3', pip install openmdao[doe]") def test_full_factorial(self): prob = om.Problem() model = prob.model @@ -566,7 +559,7 @@ def test_full_factorial(self): for name in ('x', 'y', 'f_xy'): self.assertEqual(outputs[name], expected_case[name]) - @unittest.skipUnless(pyDOE3, "requires 'pyDOE3', install openmdao[doe]") + @unittest.skipUnless(pyDOE3, "requires 'pyDOE3', pip install openmdao[doe]") def test_full_factorial_factoring(self): class Digits2Num(om.ExplicitComponent): @@ -613,7 +606,7 @@ def compute(self, inputs, outputs): # number of cases self.assertEqual(len(set(objs)), 16) - @unittest.skipUnless(pyDOE3, "requires 'pyDOE3', install openmdao[doe]") + @unittest.skipUnless(pyDOE3, "requires 'pyDOE3', pip install openmdao[doe]") def test_full_factorial_array(self): prob = om.Problem() model = prob.model @@ -655,7 +648,7 @@ def test_full_factorial_array(self): self.assertEqual(outputs['xy'][0], expected_case['xy'][0]) self.assertEqual(outputs['xy'][1], expected_case['xy'][1]) - @unittest.skipUnless(pyDOE3, "requires 'pyDOE3', install openmdao[doe]") + @unittest.skipUnless(pyDOE3, "requires 'pyDOE3', pip install openmdao[doe]") def test_full_fact_dict_levels(self): # Specifying levels only for one DV, the other is defaulted prob = om.Problem() @@ -697,7 +690,7 @@ def test_full_fact_dict_levels(self): self.assertEqual(outputs['y'], expected_case['y']) self.assertEqual(outputs['f_xy'], expected_case['f_xy']) - @unittest.skipUnless(pyDOE3, "requires 'pyDOE3', install openmdao[doe]") + @unittest.skipUnless(pyDOE3, "requires 'pyDOE3', pip install openmdao[doe]") def test_generalized_subset(self): # All DVs have the same number of levels prob = om.Problem() @@ -733,7 +726,7 @@ def test_generalized_subset(self): for name in ('x', 'y', 'f_xy'): self.assertEqual(outputs[name], expected_case[name]) - @unittest.skipUnless(pyDOE3, "requires 'pyDOE3', install openmdao[doe]") + @unittest.skipUnless(pyDOE3, "requires 'pyDOE3', pip install openmdao[doe]") def test_generalized_subset_dict_levels(self): # Number of variables specified individually for all DVs (scalars). prob = om.Problem() @@ -776,7 +769,7 @@ def test_generalized_subset_dict_levels(self): for name in ('x', 'y', 'f_xy'): self.assertAlmostEqual(outputs[name][0], expected_case[name][0]) - @unittest.skipUnless(pyDOE3, "requires 'pyDOE3', install openmdao[doe]") + @unittest.skipUnless(pyDOE3, "requires 'pyDOE3', pip install openmdao[doe]") def test_generalized_subset_array(self): # Number of levels specified individually for all DVs (arrays). @@ -823,7 +816,7 @@ def compute(self, inputs, outputs): # Testing uniqueness. If all elements are unique, it should be the same length as the number of cases self.assertEqual(len(set(objs)), 104) - @unittest.skipUnless(pyDOE3, "requires 'pyDOE3', install openmdao[doe]") + @unittest.skipUnless(pyDOE3, "requires 'pyDOE3', pip install openmdao[doe]") def test_plackett_burman(self): prob = om.Problem() model = prob.model @@ -860,7 +853,7 @@ def test_plackett_burman(self): for name in ('x', 'y', 'f_xy'): self.assertEqual(outputs[name], expected_case[name]) - @unittest.skipUnless(pyDOE3, "requires 'pyDOE3', install openmdao[doe]") + @unittest.skipUnless(pyDOE3, "requires 'pyDOE3', pip install openmdao[doe]") def test_box_behnken(self): upper = 10. center = 1 @@ -921,7 +914,7 @@ def test_box_behnken(self): for name in ('x', 'y', 'z'): self.assertEqual(outputs[name], expected_case[name]) - @unittest.skipUnless(pyDOE3, "requires 'pyDOE3', install openmdao[doe]") + @unittest.skipUnless(pyDOE3, "requires 'pyDOE3', pip install openmdao[doe]") def test_latin_hypercube(self): samples = 4 @@ -994,7 +987,7 @@ def test_latin_hypercube(self): self.assertEqual(x_buckets_filled, all_buckets) self.assertEqual(y_buckets_filled, all_buckets) - @unittest.skipUnless(pyDOE3, "requires 'pyDOE3', install openmdao[doe]") + @unittest.skipUnless(pyDOE3, "requires 'pyDOE3', pip install openmdao[doe]") def test_latin_hypercube_array(self): samples = 4 @@ -1063,7 +1056,7 @@ def test_latin_hypercube_array(self): self.assertEqual(x_buckets_filled, all_buckets) self.assertEqual(y_buckets_filled, all_buckets) - @unittest.skipUnless(pyDOE3, "requires 'pyDOE3', install openmdao[doe]") + @unittest.skipUnless(pyDOE3, "requires 'pyDOE3', pip install openmdao[doe]") def test_latin_hypercube_center(self): samples = 4 upper = 10. @@ -1122,7 +1115,7 @@ def test_latin_hypercube_center(self): self.assertEqual(x_buckets_filled, all_buckets) self.assertEqual(y_buckets_filled, all_buckets) - @unittest.skipUnless(pyDOE3, "requires 'pyDOE3', install openmdao[doe]") + @unittest.skipUnless(pyDOE3, "requires 'pyDOE3', pip install openmdao[doe]") def test_record_bug(self): # There was a bug that caused values to be recorded in driver_scaled form. @@ -1382,7 +1375,7 @@ def test_discrete_desvar_csv(self): self.assertEqual(outputs[name], expected_case[name]) self.assertTrue(isinstance(outputs[name], int)) - @unittest.skipUnless(pyDOE3, "requires 'pyDOE3', install openmdao[doe]") + @unittest.skipUnless(pyDOE3, "requires 'pyDOE3', pip install openmdao[doe]") def test_desvar_indices(self): prob = om.Problem() prob.model.add_subsystem('comp', om.ExecComp('y=x**2', @@ -1438,7 +1431,7 @@ def test_multidimensional_inputs(self): for name in ('x', 'y', 'z'): assert_near_equal(outputs[name], prob[name]) - @unittest.skipUnless(pyDOE3, "requires 'pyDOE3', install openmdao[doe]") + @unittest.skipUnless(pyDOE3, "requires 'pyDOE3', pip install openmdao[doe]") def test_multi_constraint_doe(self): prob = om.Problem() prob.model.add_subsystem('comp', om.ExecComp('y=x**2 + b', @@ -1500,7 +1493,7 @@ def test_derivative_recording(self): for dv in ('x', 'y'): self.assertEqual(derivs['f_xy', dv], expected_deriv['f_xy', dv]) - @unittest.skipUnless(pyDOE3, "requires 'pyDOE3', install openmdao[doe]") + @unittest.skipUnless(pyDOE3, "requires 'pyDOE3', pip install openmdao[doe]") def test_derivative_scaled_recording(self): prob = om.Problem() model = prob.model @@ -2192,7 +2185,7 @@ def test_full_factorial(self): @unittest.skipUnless(MPI and PETScVector, "MPI and PETSc are required.") -@unittest.skipUnless(pyDOE3, "requires 'pyDOE3', install openmdao[doe]") +@unittest.skipUnless(pyDOE3, "requires 'pyDOE3', pip install openmdao[doe]") @use_tempdirs class TestParallelDistribDOE(unittest.TestCase): diff --git a/openmdao/drivers/tests/test_pyoptsparse_driver.py b/openmdao/drivers/tests/test_pyoptsparse_driver.py index ffc4bcdce7..17e8cfdc13 100644 --- a/openmdao/drivers/tests/test_pyoptsparse_driver.py +++ b/openmdao/drivers/tests/test_pyoptsparse_driver.py @@ -2747,6 +2747,39 @@ def test_dynamic_coloring_w_multi_constraints(self): assert_near_equal(p.get_val('exec.z')[0], 25, tolerance=1e-4) assert_near_equal(p.get_val('exec.z')[50], -75, tolerance=1e-4) + def test_prom_names(self): + + p = om.Problem() + + exec = om.ExecComp(['y = a*x**2', + 'z = a + x**2'], + a={'shape': (1,)}, + y={'shape': (101,)}, + x={'shape': (101,)}, + z={'shape': (101,)}) + + p.model.add_subsystem('exec', exec, + promotes_inputs=['a', 'x'], + promotes_outputs=['y', 'z']) + + p.model.add_design_var('a', lower=-1000, upper=1000) + p.model.add_objective('y', index=50) + p.model.add_constraint('z', indices=[0], equals=25) + p.model.add_constraint('z', indices=[-1], lower=20, alias="ALIAS_TEST") + + p.driver = om.pyOptSparseDriver(optimizer=OPTIMIZER) + + p.setup(mode='rev') + p.set_val('x', np.linspace(-10, 10, 101)) + p.run_driver() + + # check that the pyoptsparse solution uses promoted names + sol = p.driver.pyopt_solution + + self.assertEqual(set(sol.variables.keys()), {'a'}) + self.assertEqual(set(sol.objectives.keys()), {'y'}) + self.assertEqual(set(sol.constraints.keys()), {'z', 'ALIAS_TEST'}) + def test_hist_file_hotstart(self): filename = "hist_file" diff --git a/openmdao/recorders/tests/test_sqlite_recorder.py b/openmdao/recorders/tests/test_sqlite_recorder.py index c24065a388..82391018b0 100644 --- a/openmdao/recorders/tests/test_sqlite_recorder.py +++ b/openmdao/recorders/tests/test_sqlite_recorder.py @@ -2703,12 +2703,14 @@ def test_problem_record_before_final_setup(self): prob.add_recorder(self.recorder) prob.setup() - msg = "Problem .*: Problem.record\(\) cannot be called before " \ - "`Problem\.run_model\(\)`, `Problem\.run_driver\(\)`, or " \ - "`Problem\.final_setup\(\)`\." - with self.assertRaisesRegex(RuntimeError, msg): + msg = f"Problem {prob._get_inst_id()}: Problem.record() cannot be called before " \ + "`Problem.run_model()`, `Problem.run_driver()`, or `Problem.final_setup()`." + + with self.assertRaises(RuntimeError) as cm: prob.record('initial') + self.assertEqual(str(cm.exception), msg) + prob.cleanup() def test_cobyla_constraints(self): diff --git a/openmdao/utils/coloring.py b/openmdao/utils/coloring.py index d97890793b..2cfffc729c 100644 --- a/openmdao/utils/coloring.py +++ b/openmdao/utils/coloring.py @@ -3,7 +3,6 @@ """ import datetime import io -import itertools import os import time import pickle @@ -23,7 +22,7 @@ from openmdao.core.constants import INT_DTYPE, _DEFAULT_OUT_STREAM from openmdao.utils.general_utils import _src_or_alias_dict, \ - _src_name_iter, _src_or_alias_item_iter + _src_name_iter, _src_or_alias_item_iter, _convert_auto_ivc_to_conn_name import openmdao.utils.hooks as hooks from openmdao.utils.mpi import MPI from openmdao.utils.file_utils import _load_and_exec, image2html @@ -154,6 +153,8 @@ class Coloring(object): Names of total jacobian rows or columns. _local_array : ndarray or None: Indices of total jacobian rows or columns. + _abs2prom : {'input': dict, 'output': dict} + Dictionary mapping absolute names to promoted names. """ def __init__(self, sparsity, row_vars=None, row_var_sizes=None, col_vars=None, @@ -187,6 +188,8 @@ def __init__(self, sparsity, row_vars=None, row_var_sizes=None, col_vars=None, self._names_array = {'fwd': None, 'rev': None} self._local_array = {'fwd': None, 'rev': None} + self._abs2prom = None + def color_iter(self, direction): """ Given a direction, yield an iterator over column (or row) groups. @@ -596,7 +599,7 @@ def __repr__(self): return ( f"Coloring (direction: {direction}, ncolors: {self.total_solves()}, shape: {shape}" - f", pct nonzero: {self._pct_nonzero:.2f}, tol: {self._meta.get('good_tol')}" + f", pct nonzero: {self._pct_nonzero:.2f}, tol: {self._meta.get('good_tol')})" ) def summary(self, out_stream=_DEFAULT_OUT_STREAM): @@ -657,7 +660,8 @@ def summary(self, out_stream=_DEFAULT_OUT_STREAM): if coloring_timestamp is not None: print(f"Coloring created on: {coloring_timestamp}", file=out_stream) - def display_txt(self, out_stream=_DEFAULT_OUT_STREAM, html=False, summary=True): + def display_txt(self, out_stream=_DEFAULT_OUT_STREAM, html=False, summary=True, + use_prom_names=True): """ Print the structure of a boolean array with coloring info for each nonzero value. @@ -679,6 +683,8 @@ def display_txt(self, out_stream=_DEFAULT_OUT_STREAM, html=False, summary=True): be plain text. summary : bool If True, include the coloring summary. + use_prom_names : bool + If True, display promoted names rather than absolute path names for variables. """ if out_stream == _DEFAULT_OUT_STREAM: out_stream = sys.stdout @@ -753,7 +759,12 @@ def display_txt(self, out_stream=_DEFAULT_OUT_STREAM, html=False, summary=True): for c in range(colstart, colend): print(charr[r, c], end='', file=out_stream) colstart = colend - print(' %d %s' % (r, rv), file=out_stream) # include row variable with row + if use_prom_names and self._abs2prom: + row_var_name = self._get_prom_name(rv) + else: + row_var_name = rv + # include row variable with row + print(' %d %s' % (r, row_var_name), file=out_stream) rowstart = rowend # now print the column vars below the matrix, with each one spaced over to line up @@ -762,7 +773,10 @@ def display_txt(self, out_stream=_DEFAULT_OUT_STREAM, html=False, summary=True): for name, size in zip(self._col_vars, self._col_var_sizes): tab = ' ' * start - col_var_name = name + if use_prom_names and self._abs2prom: + col_var_name = self._get_prom_name(name) + else: + col_var_name = name print('%s|%s' % (tab, col_var_name), file=out_stream) start += size @@ -965,7 +979,8 @@ def on_resize(event): pyplot.close(fig) - def display_bokeh(source, output_file='total_coloring.html', show=False, max_colors=200): + def display_bokeh(source, output_file='total_coloring.html', show=False, max_colors=200, + use_prom_names=True): """ Display a plot of the sparsity pattern, showing grouping by color. @@ -985,6 +1000,8 @@ def display_bokeh(source, output_file='total_coloring.html', show=False, max_col to some default length, otherwise both forward and reverse displays may share shades very near white and be difficult to distinguish. Once the number of forward or reverse solves exceeds this threshold, the color pattern restarts. + use_prom_names : bool + If True, display promoted names rather than absolute path names for variables. """ if bokeh_resources is None: print("bokeh is not installed so this coloring viewer is not available. The ascii " @@ -1131,11 +1148,19 @@ def display_bokeh(source, output_file='total_coloring.html', show=False, max_col desvar_name = coloring._col_vars[np.digitize(col_idx, desvar_idx_bins)] desvar_col_map[desvar_name].add(col_idx) + if use_prom_names and coloring._abs2prom: + desvar_col_map = {coloring._get_prom_name(k): v + for k, v in desvar_col_map.items()} + resvar_col_map = {varname: set() for varname in coloring._row_vars} for row_idx in range(nrows): resvar_name = coloring._row_vars[np.digitize(row_idx, response_idx_bins)] resvar_col_map[resvar_name].add(row_idx) + if use_prom_names and coloring._abs2prom: + resvar_col_map = {coloring._get_prom_name(k): v + for k, v in resvar_col_map.items()} + design_var_js = CustomJSHover(code=""" for (var name in varnames_map) { if (varnames_map[name].has(special_vars.snap_x)) { @@ -1445,6 +1470,24 @@ def tangent_matrix(self, direction, trans=None): return tangent + def _get_prom_name(self, abs_name): + """ + Get promoted name for specified variable. + """ + abs2prom = self._abs2prom + + # if we don't have prom names, just return abs name + if not abs2prom: + return abs_name + + # if we can't find a prom name, just return abs name + if abs_name in abs2prom['input']: + return abs2prom['input'][abs_name] + elif abs_name in abs2prom['output']: + return abs2prom['output'][abs_name] + else: + return abs_name + def _order_by_ID(col_adj_matrix): """ @@ -2242,12 +2285,22 @@ def compute_total_coloring(problem, mode=None, of=None, wrt=None, # save metadata we used to create the coloring coloring._meta.update(sparsity_info) - system = problem.model if fname is not None: - if ((system._full_comm is not None and system._full_comm.rank == 0) or - (system._full_comm is None and system.comm.rank == 0)): + if ((model._full_comm is not None and model._full_comm.rank == 0) or + (model._full_comm is None and model.comm.rank == 0)): coloring.save(fname) + # save a copy of the abs2prom dict on the coloring object + # so promoted names can be used when displaying coloring data + # (also map auto_ivc names to the prom name of their connected input) + if coloring is not None: + coloring._abs2prom = abs2prom = model._var_allprocs_abs2prom.copy() + conns = model._conn_global_abs_in2out + for abs_out in abs2prom['output']: + if abs_out.startswith('_auto_ivc.'): + abs_in = _convert_auto_ivc_to_conn_name(conns, abs_out) + abs2prom['output'][abs_out] = abs2prom['input'][abs_in] + driver._total_jac = None # if we're running under MPI, make sure the coloring object is identical on all ranks diff --git a/openmdao/visualization/opt_report/opt_report.py b/openmdao/visualization/opt_report/opt_report.py index 552466e04f..582cfee3fb 100644 --- a/openmdao/visualization/opt_report/opt_report.py +++ b/openmdao/visualization/opt_report/opt_report.py @@ -130,16 +130,7 @@ def opt_report(prob, outfile=None): driver_scaling = True - # Collect data from the problem - abs2prom = prob.model._var_abs2prom - - def get_prom_name(abs_name): - if abs_name in abs2prom['input']: - return abs2prom['input'][abs_name] - elif abs_name in abs2prom['output']: - return abs2prom['output'][abs_name] - else: - return abs_name + get_prom_name = prob.model._get_prom_name # Collect the entire array of array valued desvars and constraints (ignore indices) objs_vals = {}