Merge pull request #3066 from swryan/3017_prom_names

Updated pyOptSparseDriver and Coloring to use promoted names

openmdao/core/group.py

@@ -5099,3 +5099,39 @@ def _gather_full_data(self):
                     self._subsystems_myproc[0]._full_comm.rank == 0
 
         return False
+
+    def _get_prom_name(self, abs_name):
+        """
+        Get promoted name for specified variable.
+        """
+        abs2prom = self._var_allprocs_abs2prom
+        if abs_name in abs2prom['input']:
+            return abs2prom['input'][abs_name]
+        elif abs_name in abs2prom['output']:
+            return abs2prom['output'][abs_name]
+        else:
+            return abs_name
+
+    def _prom_names_list(self, lst):
+        """
+        Convert a list of variable names to promoted names.
+        """
+        return [self._get_prom_name(n) for n in lst]
+
+    def _prom_names_dict(self, dct):
+        """
+        Convert a dictionary keyed on variable names to be keyed on promoted names.
+        """
+        return {self._get_prom_name(k): v for k, v in dct.items()}
+
+    def _prom_names_jac(self, jac):
+        """
+        Convert a nested dict jacobian keyed on variable names to be keyed on promoted names.
+        """
+        new_jac = {}
+        for of in jac:
+            new_jac[self._get_prom_name(of)] = of_dict = {}
+            for wrt in jac[of]:
+                of_dict[self._get_prom_name(wrt)] = jac[of][wrt]
+
+        return new_jac

openmdao/core/tests/test_coloring.py

@@ -125,7 +125,7 @@ def run_opt(driver_class, mode, assemble_type=None, color_info=None, derivs=True
 
     p.model.add_subsystem('arctan_yox', arctan_yox)
 
-    p.model.add_subsystem('circle', om.ExecComp('area=pi*r**2'))
+    p.model.add_subsystem('circle', om.ExecComp('area=pi*r**2'), promotes_outputs=['area'])
 
     p.model.add_subsystem('r_con', om.ExecComp('g=x**2 + y**2 - r', has_diag_partials=has_diag_partials,
                                                g=np.ones(SIZE), x=np.ones(SIZE), y=np.ones(SIZE)))
@@ -243,7 +243,7 @@ def compute(self, inputs, outputs):
             # linear constraint (if has_lin_constraint is set)
             p.model.add_constraint('y', equals=0, indices=[0,], linear=has_lin_constraint)
 
-        p.model.add_objective('circle.area', ref=-1)
+        p.model.add_objective('area', ref=-1)
 
     # setup coloring
     if color_info is not None:
@@ -312,6 +312,32 @@ def test_dynamic_total_coloring_snopt_auto_autoivc(self):
         self.assertEqual(p.model._solve_count, 21)
         self.assertEqual(p_color.model._solve_count, 5)
 
+    @unittest.skipUnless(OPTIMIZER == 'SNOPT', "This test requires SNOPT.")
+    def test_dynamic_total_coloring_display_txt(self):
+        p_color = run_opt(pyOptSparseDriver, 'auto', optimizer='SNOPT', print_results=False,
+                          dynamic_total_coloring=True, auto_ivc=True)
+
+        # check text visualizion of coloring
+        coloring = compute_total_coloring(p_color)
+
+        stream = StringIO()
+        coloring.display_txt(out_stream=stream, use_prom_names=False)
+        color_txt = stream.getvalue().split('\n')
+
+        self.assertTrue(color_txt[0].endswith('  circle.area'))
+        self.assertEqual(color_txt[22].strip(), '|_auto_ivc.v0')
+        self.assertEqual(color_txt[23].strip(), '|_auto_ivc.v1')
+        self.assertEqual(color_txt[24].strip(), '|_auto_ivc.v2')
+
+        stream = StringIO()
+        coloring.display_txt(out_stream=stream)           # use_prom_names=True is the default
+        color_txt = stream.getvalue().split('\n')
+
+        self.assertTrue(color_txt[0].endswith('  area'))  # promoted name
+        self.assertEqual(color_txt[22].strip(), '|x')     # connected input rather than _auto_ivc.v0
+        self.assertEqual(color_txt[23].strip(), '|y')     # connected input rather than _auto_ivc.v1
+        self.assertEqual(color_txt[24].strip(), '|r')     # connected input rather than _auto_ivc.v2
+
     @unittest.skipUnless(OPTIMIZER == 'SNOPT', "This test requires SNOPT.")
     def test_dynamic_total_coloring_snopt_auto_dyn_partials(self):
         # first, run w/o coloring
@@ -540,7 +566,7 @@ def test_dynamic_total_coloring_pyoptsparse_slsqp_auto(self):
 
         # test __repr__
         rep = repr(p_color.driver._coloring_info['coloring'])
-        self.assertEqual(rep.replace('L', ''), 'Coloring (direction: fwd, ncolors: 5, shape: (22, 21), pct nonzero: 13.42, tol: 1e-15')
+        self.assertEqual(rep.replace('L', ''), 'Coloring (direction: fwd, ncolors: 5, shape: (22, 21), pct nonzero: 13.42, tol: 1e-15)')
 
     @unittest.skipUnless(OPTIMIZER == 'SNOPT', "This test requires SNOPT.")
     def test_print_options_total_with_coloring_fwd(self):
@@ -821,7 +847,7 @@ def test_problem_total_coloring_auto(self):
         p = run_opt(om.ScipyOptimizeDriver, 'auto', optimizer='SLSQP', disp=False, use_vois=False)
         coloring = compute_total_coloring(p,
                                           of=['r_con.g', 'theta_con.g', 'delta_theta_con.g',
-                                              'l_conx.g', 'y', 'circle.area'],
+                                              'l_conx.g', 'y', 'area'],
                                           wrt=['x', 'y', 'r'])
         self.assertEqual(coloring.total_solves(), 5)
 
@@ -1068,12 +1094,12 @@ def test_repr(self):
         p_color = run_opt(om.ScipyOptimizeDriver, 'auto', optimizer='SLSQP', disp=False, dynamic_total_coloring=True)
         coloring = p_color.driver._coloring_info['coloring']
         rep = repr(coloring)
-        self.assertEqual(rep.replace('L', ''), 'Coloring (direction: fwd, ncolors: 5, shape: (22, 21), pct nonzero: 13.42, tol: 1e-15')
+        self.assertEqual(rep.replace('L', ''), 'Coloring (direction: fwd, ncolors: 5, shape: (22, 21), pct nonzero: 13.42, tol: 1e-15)')
 
         dense_J = np.ones((50, 50), dtype=bool)
         coloring = _compute_coloring(dense_J, 'auto')
         rep = repr(coloring)
-        self.assertEqual(rep.replace('L', ''), 'Coloring (direction: fwd, ncolors: 50, shape: (50, 50), pct nonzero: 100.00, tol: None')
+        self.assertEqual(rep.replace('L', ''), 'Coloring (direction: fwd, ncolors: 50, shape: (50, 50), pct nonzero: 100.00, tol: None)')
 
     def test_bad_mode(self):
         p_color_rev = run_opt(om.ScipyOptimizeDriver, 'rev', optimizer='SLSQP', disp=False, dynamic_total_coloring=True)

openmdao/core/tests/test_prob_remote.py

@@ -81,21 +81,29 @@ def test_is_local(self):
         par.add_subsystem('C2', om.ExecComp('y=3*x'))
         p.model.connect('indep.x', ['par.C1.x', 'par.C2.x'])
 
-        with self.assertRaisesRegex(RuntimeError,
-            "Problem .*: is_local\('indep\.x'\) was called before setup\(\) completed\."):
-            loc = p.is_local('indep.x')
-
-        with self.assertRaisesRegex(RuntimeError,
-            "Problem .*: is_local\('par\.C1'\) was called before setup\(\) completed\."):
-            loc = p.is_local('par.C1')
-
-        with self.assertRaisesRegex(RuntimeError,
-            "Problem .*: is_local\('par\.C1\.y'\) was called before setup\(\) completed\."):
-            loc = p.is_local('par.C1.y')
-
-        with self.assertRaisesRegex(RuntimeError,
-            "Problem .*: is_local\('par\.C1\.x'\) was called before setup\(\) completed\."):
-            loc = p.is_local('par.C1.x')
+        with self.assertRaises(RuntimeError) as err:
+            p.is_local('indep.x')
+        self.assertEqual(str(err.exception),
+                         f"Problem {p._get_inst_id()}: is_local('indep.x') "
+                         "was called before setup() completed.")
+
+        with self.assertRaises(RuntimeError) as err:
+            p.is_local('par.C1')
+        self.assertEqual(str(err.exception),
+                         f"Problem {p._get_inst_id()}: is_local('par.C1') "
+                         "was called before setup() completed.")
+
+        with self.assertRaises(RuntimeError) as err:
+            p.is_local('par.C1.y')
+        self.assertEqual(str(err.exception),
+                         f"Problem {p._get_inst_id()}: is_local('par.C1.y') "
+                         "was called before setup() completed.")
+
+        with self.assertRaises(RuntimeError) as err:
+            p.is_local('par.C1.x')
+        self.assertEqual(str(err.exception),
+                         f"Problem {p._get_inst_id()}: is_local('par.C1.x') "
+                         "was called before setup() completed.")
 
         p.setup()
         p.final_setup()

openmdao/core/tests/test_problem.py

@@ -739,17 +739,23 @@ def test_set_cs_error_messages(self):
         prob.model.add_subsystem('comp', Paraboloid())
         prob.setup()
         prob.run_model()
-        msg = "Problem .*: To enable complex step, specify 'force_alloc_complex=True' when calling " + \
-            "setup on the problem, e\.g\. 'problem\.setup\(force_alloc_complex=True\)'"
-        with self.assertRaisesRegex(RuntimeError, msg):
+
+        msg = f"Problem {prob._get_inst_id()}: To enable complex step, specify 'force_alloc_complex=True' when calling " + \
+              "setup on the problem, e.g. 'problem.setup(force_alloc_complex=True)'"
+
+        with self.assertRaises(RuntimeError) as err:
             prob.set_complex_step_mode(True)
+        self.assertEqual(str(err.exception), msg)
 
         prob = om.Problem()
         prob.model.add_subsystem('comp', Paraboloid())
-        msg = "Problem .*: set_complex_step_mode cannot be called before `Problem\.run_model\(\)`, " + \
-            "`Problem\.run_driver\(\)`, or `Problem\.final_setup\(\)`."
-        with self.assertRaisesRegex(RuntimeError, msg) :
+
+        msg = f"Problem {prob._get_inst_id()}: set_complex_step_mode cannot be called before `Problem.run_model()`, " + \
+              "`Problem.run_driver()`, or `Problem.final_setup()`."
+
+        with self.assertRaises(RuntimeError) as err:
             prob.set_complex_step_mode(True)
+        self.assertEqual(str(err.exception), msg)
 
     def test_feature_run_driver(self):
 
@@ -1818,19 +1824,23 @@ def test_list_problem_vars_before_final_setup(self):
         prob.model.add_subsystem('const', om.ExecComp('g = x + y'), promotes_inputs=['x', 'y'])
         prob.model.set_input_defaults('x', 3.0)
         prob.model.set_input_defaults('y', -4.0)
+
         prob.driver = om.ScipyOptimizeDriver()
         prob.driver.options['optimizer'] = 'COBYLA'
+
         prob.model.add_design_var('x', lower=-50, upper=50)
         prob.model.add_design_var('y', lower=-50, upper=50)
         prob.model.add_objective('parab.f_xy')
         prob.model.add_constraint('const.g', lower=0, upper=10.)
+
         prob.setup()
 
-        msg = "Problem .*: Problem.list_problem_vars\(\) cannot be called before " \
-                         "`Problem\.run_model\(\)`, `Problem\.run_driver\(\)`, or " \
-                         "`Problem\.final_setup\(\)`\."
-        with self.assertRaisesRegex(RuntimeError, msg):
+        msg = f"Problem {prob._get_inst_id()}: Problem.list_problem_vars() cannot be called " \
+              "before `Problem.run_model()`, `Problem.run_driver()`, or `Problem.final_setup()`."
+
+        with self.assertRaises(RuntimeError) as err:
             prob.list_problem_vars()
+        self.assertEqual(str(err.exception), msg)
 
     def test_list_problem_w_multi_constraints(self):
         p = om.Problem()

openmdao/docs/openmdao_book/features/core_features/working_with_derivatives/simul_derivs.ipynb

@@ -34,7 +34,7 @@
     "\n",
     "When mode is set to ‘fwd’ or ‘rev’, a unidirectional coloring algorithm is used to group columns or rows, respectively, for simultaneous derivative calculation. The algorithm used in this case is the greedy algorithm with ordering by incidence degree found in T. F. Coleman and J. J. More, *Estimation of sparse Jacobian matrices and graph coloring problems*, SIAM J. Numer. Anal., 20 (1983), pp. 187–209.\n",
     "\n",
-    "When using simultaneous derivatives, setting mode=’auto’ will indicate that bidirectional coloring should be used. Bidirectional coloring can significantly decrease the number of linear solves needed to generate the total Jacobian relative to coloring only in fwd or rev mode.\n",
+    "When using simultaneous derivatives, setting `mode=’auto’` will indicate that bidirectional coloring should be used. Bidirectional coloring can significantly decrease the number of linear solves needed to generate the total Jacobian relative to coloring only in fwd or rev mode.\n",
     "\n",
     "For more information on the bidirectional coloring algorithm, see T. F. Coleman and A. Verma, *The efficient computation of sparse Jacobian matrices using automatic differentiation*, SIAM J. Sci. Comput., 19 (1998), pp. 1210–1233."
    ]
@@ -248,14 +248,17 @@
    "id": "954218f7",
    "metadata": {},
    "source": [
-    "(view_coloring-api)=\n",
-    "## openmdao.api.view_coloring\n",
+    "(display_coloring-api)=\n",
+    "## Viewing total coloring from a script\n",
     "\n",
-    "The `openmdao.api` includes a view_coloring command to allow viewing of coloring data from within a script.\n",
-    "This command will generate an html file, but will render the coloring as plain text if bokeh is unavailable or if the `as_text` argument is True.\n",
+    "The `openmdao.api` includes a `display_coloring` function to allow viewing of coloring data from within a script.\n",
+    "This function will generate an html file, but will render the coloring as plain text if bokeh is unavailable or if the `as_text` argument is True.\n",
     "\n",
-    ".. autofunction:: openmdao.utils.coloring.view_coloring\n",
-    "    :noindex:"
+    "```{eval-rst}\n",
+    "    .. autofunction:: openmdao.utils.coloring.display_coloring\n",
+    "        :noindex:\n",
+    "```\n",
+    "\n"
    ]
   },
   {

openmdao/docs/openmdao_book/theory_manual/advanced_linear_solvers_special_cases/separable.ipynb

@@ -90,7 +90,7 @@
     "Hence, it would be faster to solve for total derivatives using forward mode with simultaneous\n",
     "derivatives than reverse mode.\n",
     "\n",
-    "Determining if Your Problem is Separable\n",
+    "## Determining if Your Problem is Separable\n",
     "\n",
     "The simple example above was contrived to make it relatively obvious that the problem was separable.\n",
     "For realistic problems, even if you know that the problem should be separable, computing the actual\n",
@@ -123,7 +123,8 @@
     "If the model is intended to be used in an optimization context, then it is fair to assume that the total-derivative Jacobian is inexpensive enough to compute many times,\n",
     "and using a few additional computations to compute a coloring will not significantly impact the overall compute cost.\n",
     "\n",
-    "Choosing Forward or Reverse Mode for Separable Problems\n",
+    "## Choosing Forward or Reverse Mode for Separable Problems\n",
+    "\n",
     "If a problem has a section of design variables and constraints that are separable,\n",
     "then it is possible to leverage that quality in either forward or reverse mode.\n",
     "Which mode you choose depends on which direction gives you fewer total linear solves.\n",
@@ -134,12 +135,13 @@
     "Sometimes the answer is different than you would get by counting design variables and constraints, but sometimes its not.\n",
     "The result is problem-dependent.\n",
     "\n",
-    "Relevance to Finite Difference and Complex Step\n",
+    "## Relevance to Finite Difference and Complex Step\n",
+    "\n",
     "It is worth noting that, in addition to speeding up linear solutions for the unified derivative equations, forward separability also offers benefits when finite difference or complex step are being used to compute derivatives numerically.\n",
     "For the same reasons that multiple linear solves can be combined, you can also take steps in multiple variables to compute derivatives with respect to multiple variables at the same time.\n",
     "\n",
+    "## How to actually use it!\n",
     "\n",
-    "How to actually use it!\n",
     "OpenMDAO provides a mechanism for you to specify a coloring to take advantage of separability, via the\n",
     "[use_fixed_coloring](../../features/core_features/working_with_derivatives/simul_derivs.ipynb) method.\n",
     "OpenMDAO also provides a [coloring tool](static-coloring) to determine the minimum number of colors your problem can be reduced to.\n",

openmdao/drivers/doe_driver.py

@@ -34,9 +34,9 @@ class DOEDriver(Driver):
     _color : int or None
         In MPI, the cached color is used to determine which cases to run on this proc.
     _indep_list : list
-        List of design variables.
+        List of design variables, used to compute derivatives.
     _quantities : list
-        Contains the objectives plus nonlinear constraints.
+        Contains the objectives plus nonlinear constraints, used to compute derivatives.
     """
 
     def __init__(self, generator=None, **kwargs):
@@ -232,8 +232,7 @@ def _run_case(self, case):
             # save reference to metadata for use in record_iteration
             self._metadata = metadata
 
-        opts = self.recording_options
-        if opts['record_derivatives']:
+        if self.recording_options['record_derivatives']:
             self._compute_totals(of=self._quantities,
                                  wrt=self._indep_list,
                                  return_format=self._total_jac_format,