Merge pull request #1089 from naylor-b/workshop

command line updates

openmdao/core/driver.py

@@ -972,39 +972,6 @@ def _get_name(self):
         """
         return "Driver"
 
-    def set_total_jac_sparsity(self, sparsity):
-        """
-        Set the sparsity of sub-jacobians of the total jacobian.
-
-        Note: This currently will have no effect if you are not using the pyOptSparseDriver.
-
-        Parameters
-        ----------
-        sparsity : str or dict
-
-            ::
-
-                # Sparsity is a nested dictionary where the outer keys are response
-                # names, the inner keys are design variable names, and the value is a tuple of
-                # the form (row_list, col_list, shape).
-                {
-                    resp1: {
-                        dv1: (rows, cols, shape),  # for sub-jac d_resp1/d_dv1
-                        dv2: (rows, cols, shape),
-                          ...
-                    },
-                    resp2: {
-                        ...
-                    }
-                    ...
-                }
-        """
-        if self.supports['total_jac_sparsity']:
-            self._total_jac_sparsity = sparsity
-        else:
-            raise RuntimeError("Driver '%s' does not support setting of total jacobian sparsity." %
-                               self._get_name())
-
     def declare_coloring(self, num_full_jacs=coloring_mod._DEF_COMP_SPARSITY_ARGS['num_full_jacs'],
                          tol=coloring_mod._DEF_COMP_SPARSITY_ARGS['tol'],
                          orders=coloring_mod._DEF_COMP_SPARSITY_ARGS['orders'],

openmdao/core/tests/test_check_derivs.py

@@ -2379,7 +2379,7 @@ def test_cs_error_allocate(self):
         with self.assertRaises(RuntimeError) as cm:
             prob.check_totals(method='cs')
 
-        msg = "\nTo enable complex step, specify 'force_alloc_complex=True' when calling " + \
+        msg = "\nProblem: To enable complex step, specify 'force_alloc_complex=True' when calling " + \
                 "setup on the problem, e.g. 'problem.setup(force_alloc_complex=True)'"
         self.assertEqual(str(cm.exception), msg)
 

openmdao/core/tests/test_coloring.py

@@ -92,7 +92,7 @@ def compute(self, inputs, outputs):
 
 
 
-def run_opt(driver_class, mode, assemble_type=None, color_info=None, sparsity=None, derivs=True,
+def run_opt(driver_class, mode, assemble_type=None, color_info=None, derivs=True,
             recorder=None, has_lin_constraint=True, has_diag_partials=True, partial_coloring=False,
             **options):
 
@@ -178,8 +178,6 @@ def run_opt(driver_class, mode, assemble_type=None, color_info=None, sparsity=No
     # # setup coloring
     if color_info is not None:
         p.driver.use_fixed_coloring(color_info)
-    elif sparsity is not None:
-        p.driver.set_total_jac_sparsity(sparsity)
 
     if recorder:
         p.driver.add_recorder(recorder)
@@ -760,49 +758,6 @@ def setUp(self):
             }
         }
 
-    @unittest.skipUnless(OPTIMIZER == 'SNOPT', "This test requires SNOPT.")
-    def test_sparsity_snopt(self):
-        # first, run without sparsity
-        p = run_opt(pyOptSparseDriver, 'fwd', optimizer='SNOPT', print_results=False)
-
-        # run with dynamic sparsity
-        p_dynamic = run_opt(pyOptSparseDriver, 'fwd', dynamic_derivs_sparsity=True,
-                            optimizer='SNOPT', print_results=False)
-
-        # run with provided sparsity
-        p_sparsity = run_opt(pyOptSparseDriver, 'fwd', sparsity=self.sparsity,
-                             optimizer='SNOPT', print_results=False)
-
-        assert_almost_equal(p['circle.area'], np.pi, decimal=7)
-        assert_almost_equal(p_dynamic['circle.area'], np.pi, decimal=7)
-        assert_almost_equal(p_sparsity['circle.area'], np.pi, decimal=7)
-
-    def test_sparsity_pyoptsparse_slsqp(self):
-        try:
-            from pyoptsparse import OPT
-        except ImportError:
-            raise unittest.SkipTest("This test requires pyoptsparse.")
-
-        try:
-            OPT('SLSQP')
-        except:
-            raise unittest.SkipTest("This test requires pyoptsparse SLSQP.")
-
-        # first, run without sparsity
-        p = run_opt(pyOptSparseDriver, 'fwd', optimizer='SLSQP', print_results=False)
-
-        # run with dynamic sparsity
-        p_dynamic = run_opt(pyOptSparseDriver, 'fwd', dynamic_derivs_sparsity=True,
-                            optimizer='SLSQP', print_results=False)
-
-        # run with provided sparsity
-        p_sparsity = run_opt(pyOptSparseDriver, 'fwd', sparsity=self.sparsity,
-                             optimizer='SLSQP', print_results=False)
-
-        assert_almost_equal(p['circle.area'], np.pi, decimal=7)
-        assert_almost_equal(p_dynamic['circle.area'], np.pi, decimal=7)
-        assert_almost_equal(p_sparsity['circle.area'], np.pi, decimal=7)
-
 
 def _test_func_name(func, num, param):
     args = []

openmdao/core/tests/test_distribcomp.py

@@ -569,19 +569,19 @@ def test_prob_getitem_err(self):
         with self.assertRaises(Exception) as context:
             ans = p.get_val('par.C2.invec', get_remote=True)
         self.assertEqual(str(context.exception),
-                         "Retrieval of the full distributed variable 'par.C2.invec' is not supported.")
+                         "Problem: Retrieval of the full distributed variable 'par.C2.invec' is not supported.")
         with self.assertRaises(Exception) as context:
             ans = p.get_val('par.C2.outvec', get_remote=True)
         self.assertEqual(str(context.exception),
-                         "Retrieval of the full distributed variable 'par.C2.outvec' is not supported.")
+                         "Problem: Retrieval of the full distributed variable 'par.C2.outvec' is not supported.")
         with self.assertRaises(Exception) as context:
             ans = p.get_val('par.C1.invec', get_remote=True)
         self.assertEqual(str(context.exception),
-                         "Retrieval of the full distributed variable 'par.C1.invec' is not supported.")
+                         "Problem: Retrieval of the full distributed variable 'par.C1.invec' is not supported.")
         with self.assertRaises(Exception) as context:
             ans = p.get_val('par.C1.outvec', get_remote=True)
         self.assertEqual(str(context.exception),
-                         "Retrieval of the full distributed variable 'par.C1.outvec' is not supported.")
+                         "Problem: Retrieval of the full distributed variable 'par.C1.outvec' is not supported.")
 
 
 @unittest.skipUnless(PETScVector, "PETSc is required.")

openmdao/core/tests/test_parallel_groups.py

@@ -303,7 +303,7 @@ def test_setup_messages_bad_vec_type(self):
 
         # check that error is thrown if not using PETScVector
         if MPI:
-            msg = ("The `distributed_vector_class` argument must be `PETScVector` when "
+            msg = ("Problem: The `distributed_vector_class` argument must be `PETScVector` when "
                    "running in parallel under MPI but 'DefaultVector' was specified.")
             with self.assertRaises(ValueError) as cm:
                 prob.setup(check=False, mode='fwd', distributed_vector_class=om.DefaultVector)

openmdao/core/tests/test_prob_remote.py

@@ -82,19 +82,19 @@ def test_is_local(self):
 
         with self.assertRaises(RuntimeError) as cm:
             loc = p.is_local('indep.x')
-        self.assertEqual(str(cm.exception), "is_local('indep.x') was called before setup() completed.")
+        self.assertEqual(str(cm.exception), "Problem: is_local('indep.x') was called before setup() completed.")
 
         with self.assertRaises(RuntimeError) as cm:
             loc = p.is_local('par.C1')
-        self.assertEqual(str(cm.exception), "is_local('par.C1') was called before setup() completed.")
+        self.assertEqual(str(cm.exception), "Problem: is_local('par.C1') was called before setup() completed.")
 
         with self.assertRaises(RuntimeError) as cm:
             loc = p.is_local('par.C1.y')
-        self.assertEqual(str(cm.exception), "is_local('par.C1.y') was called before setup() completed.")
+        self.assertEqual(str(cm.exception), "Problem: is_local('par.C1.y') was called before setup() completed.")
 
         with self.assertRaises(RuntimeError) as cm:
             loc = p.is_local('par.C1.x')
-        self.assertEqual(str(cm.exception), "is_local('par.C1.x') was called before setup() completed.")
+        self.assertEqual(str(cm.exception), "Problem: is_local('par.C1.x') was called before setup() completed.")
 
         p.setup()
         p.final_setup()

openmdao/core/tests/test_problem.py

@@ -12,6 +12,7 @@
 from openmdao.core.system import get_relevant_vars
 from openmdao.core.driver import Driver
 from openmdao.utils.assert_utils import assert_rel_error, assert_warning
+import openmdao.utils.hooks as hooks
 from openmdao.test_suite.components.paraboloid import Paraboloid
 from openmdao.test_suite.components.sellar import SellarDerivatives
 
@@ -1006,15 +1007,15 @@ def test_get_set_with_units_error_messages(self):
         with assertRaisesRegex(self, TypeError, msg):
             prob.get_val('comp.x', 'degK')
 
-        msg = "Can't set variable 'comp.x' with units of 'cm' to value with units of 'degK'."
+        msg = "Can't set variable 'comp.x' with units 'cm' to value with units 'degK'."
         with assertRaisesRegex(self, TypeError, msg):
             prob.set_val('comp.x', 55.0, 'degK')
 
         msg = "Can't express variable 'no_unit.x' with units of 'None' in units of 'degK'."
         with assertRaisesRegex(self, TypeError, msg):
             prob.get_val('no_unit.x', 'degK')
 
-        msg = "Can't set variable 'no_unit.x' with units of 'None' to value with units of 'degK'."
+        msg = "Can't set variable 'no_unit.x' with units 'None' to value with units 'degK'."
         with assertRaisesRegex(self, TypeError, msg):
             prob.set_val('no_unit.x', 55.0, 'degK')
 
@@ -1134,12 +1135,12 @@ def test_feature_residuals(self):
     def test_setup_bad_mode(self):
         # Test error message when passing bad mode to setup.
 
-        prob = om.Problem()
+        prob = om.Problem(name='foo')
 
         try:
             prob.setup(mode='junk')
         except ValueError as err:
-            msg = "Unsupported mode: 'junk'. Use either 'fwd' or 'rev'."
+            msg = "Problem foo: Unsupported mode: 'junk'. Use either 'fwd' or 'rev'."
             self.assertEqual(str(err), msg)
         else:
             self.fail('Expecting ValueError')
@@ -1194,23 +1195,23 @@ def test_run_before_setup(self):
         try:
             prob.run_model()
         except RuntimeError as err:
-            msg = "The `setup` method must be called before `run_model`."
+            msg = "Problem: The `setup` method must be called before `run_model`."
             self.assertEqual(str(err), msg)
         else:
             self.fail('Expecting RuntimeError')
 
         try:
             prob.run_driver()
         except RuntimeError as err:
-            msg = "The `setup` method must be called before `run_driver`."
+            msg = "Problem: The `setup` method must be called before `run_driver`."
             self.assertEqual(str(err), msg)
         else:
             self.fail('Expecting RuntimeError')
 
     def test_run_with_invalid_prefix(self):
         # Test error message when running with invalid prefix.
 
-        msg = "The 'case_prefix' argument should be a string."
+        msg = "Problem: The 'case_prefix' argument should be a string."
 
         prob = om.Problem()
 
@@ -1249,7 +1250,7 @@ def test_root_deprecated(self):
             prob = om.Problem(root=om.Group(), model=om.Group())
 
         self.assertEqual(str(cm.exception),
-                         "Cannot specify both 'root' and 'model'. "
+                         "Problem: Cannot specify both 'root' and 'model'. "
                          "'root' has been deprecated, please use 'model'.")
 
         msg = "The 'root' argument provides backwards " \
@@ -1284,14 +1285,14 @@ def test_args(self):
             prob = om.Problem(om.ScipyOptimizeDriver())
 
         self.assertEqual(str(cm.exception),
-                         "The value provided for 'model' is not a valid System.")
+                         "Problem: The value provided for 'model' is not a valid System.")
 
         # invalid driver
         with self.assertRaises(TypeError) as cm:
             prob = om.Problem(driver=SellarDerivatives())
 
         self.assertEqual(str(cm.exception),
-                         "The value provided for 'driver' is not a valid Driver.")
+                         "Problem: The value provided for 'driver' is not a valid Driver.")
 
     def test_relevance(self):
         p = om.Problem()
@@ -1602,15 +1603,16 @@ def setup(self):
         self.assertTrue(isinstance(top.model.sub.nonlinear_solver, om.NewtonSolver))
         self.assertTrue(isinstance(top.model.sub.linear_solver, om.ScipyKrylov))
 
-    def test_post_setup_hook(self):
+    def test_post_final_setup_hook(self):
         def hook_func(prob):
             prob['p2.y'] = 5.0
 
-        prob = om.Problem()
-        model = prob.model
-        om.Problem._post_setup_func = hook_func
-
+        hooks.use_hooks = True
+        hooks._register_hook('final_setup', class_name='Problem', post=hook_func)
         try:
+            prob = om.Problem()
+            model = prob.model
+
             model.add_subsystem('p1', om.IndepVarComp('x', 3.0))
             model.add_subsystem('p2', om.IndepVarComp('y', -4.0))
             model.add_subsystem('comp', om.ExecComp("f_xy=2.0*x+3.0*y"))
@@ -1624,7 +1626,8 @@ def hook_func(prob):
             assert_rel_error(self, prob['p2.y'], 5.0)
             assert_rel_error(self, prob['comp.f_xy'], 21.0)
         finally:
-            om.Problem._post_setup_func = None
+            hooks._unregister_hook('final_setup', class_name='Problem')
+            hooks.use_hooks = False
 
     def test_list_problem_vars(self):
         model = SellarDerivatives()

openmdao/devtools/debug.py

@@ -133,7 +133,7 @@ def color_print(s, fore='', color='', end=''):
 
 
 def tree(top, show_solvers=True, show_jacs=True, show_colors=True, show_approx=True,
-         filter=None, max_depth=0, rank=0, stream=sys.stdout):
+         filter=None, show_sizes=False, max_depth=0, rank=0, stream=sys.stdout):
     """
     Dump the model tree structure to the given stream.
 
@@ -156,6 +156,8 @@ def tree(top, show_solvers=True, show_jacs=True, show_colors=True, show_approx=T
         A function taking a System arg and returning None or an iter of (name, value) tuples.
         If None is returned, that system will not be displayed.  Otherwise, the system will
         be displayed along with any name, value pairs returned from the filter.
+    show_sizes : bool
+        If True, show input and output sizes for each System.
     max_depth : int
         Maximum depth for display.
     rank : int
@@ -202,8 +204,9 @@ def tree(top, show_solvers=True, show_jacs=True, show_colors=True, show_approx=T
             cprint("%s" % s.name)
 
         # FIXME: these sizes could be wrong under MPI
-        cprint(" (%d / %d)" % (s._inputs._data.size, s._outputs._data.size),
-               color=Fore.RED + Style.BRIGHT)
+        if show_sizes:
+            cprint(" (%d / %d)" % (s._inputs._data.size, s._outputs._data.size),
+                color=Fore.RED + Style.BRIGHT)
 
         if show_solvers:
             lnsolver = type(s.linear_solver).__name__

openmdao/devtools/iprof_mem.py

@@ -119,17 +119,18 @@ def _list_package_pyfiles(package):
 
 def _mem_prof_setup_parser(parser):
     parser.add_argument('-o', '--outfile', action='store', dest='outfile', default='mem_trace.raw',
-                        help='Name of file containing memory dump.')
+                        help='Name of file containing memory dump. Default is mem_trace.raw.')
     parser.add_argument('--min', action='store', dest='min_mem', type=float, default=1.0,
                         help='Dump function trace with memory usage in MB above min_mem to the '
-                        'given file. Default is 1.0.')
+                        'given file. Default is 1 MB.')
     parser.add_argument('-c', '--colors', action='store_true', dest='show_colors',
                         help="Display colors if the terminal supports it.  Requires 'colorama' "
                              "python package.  Use 'pip install colorama' to install it.")
     parser.add_argument('--nogc', action='store_true', dest='nogc',
                         help="Disables automatic garbage collection.")
     parser.add_argument('-p', '--package', action='append', dest='packages',
-                        default=[], help='Determines which packages will be traced.')
+                        default=[], help='Determines which packages will be traced. '
+                        'Default package is openmdao.')
     parser.add_argument('-t', '--tree', action='store_true', dest='tree',
                         help="Display memory use in tree format, showing memory use for each "
                              "unique call sequence down to a function.")
@@ -211,7 +212,7 @@ def memtrace(**kwargs):
 
 def _mempost_setup_parser(parser):
     parser.add_argument('--out', action='store', dest='outfile', default=None,
-                        help='Dump memory tree to given file.')
+                        help='Dump memory report to given file.')
     parser.add_argument('--min', action='store', dest='min_mem', type=float, default=1.0,
                         help='Dump function trace with memory usage to given file.')
     parser.add_argument('-c', '--colors', action='store_true', dest='show_colors',

openmdao/devtools/iprofile.py

@@ -236,8 +236,6 @@ def _process_1_profile(fname):
     tree_nodes = {}
     tree_parts = []
 
-    empty = [None, 0., 0]
-
     for funcpath, count, t in _iter_raw_prof_file(fname):
         parts = funcpath.split('|')
 

openmdao/devtools/itrace.py

@@ -212,7 +212,6 @@ def _setup(options):
         method_counts = defaultdict(int)
         class_counts = defaultdict(lambda: -1)
         id2count = {}
-        do_ret = _trace_return
 
         if memory:
             if psutil is None:
@@ -241,7 +240,7 @@ def _setup(options):
 
         _trace_calls = _create_profile_callback(call_stack, _collect_methods(methods),
                                                 do_call=_trace_call,
-                                                do_ret=do_ret,
+                                                do_ret=_trace_return,
                                                 context=(qual_cache, method_counts,
                                                          class_counts, id2count, verbose, memory,
                                                          leaks, stream, options.show_ptrs),

openmdao/docs/features/core_features/working_with_derivatives/index.rst

@@ -61,7 +61,6 @@ of these features, or how to restructure your problem to make use of them in the
     :maxdepth: 1
 
     assembled_jacobian.rst
-    sparse_totals.rst
     simul_derivs.rst
     parallel_derivs.rst
     vectorized_derivs.rst