Merge 8346a0b into 691f21c

openmdao/core/driver.py

@@ -1,9 +1,11 @@
 """Define a base class for all Drivers in OpenMDAO."""
 from __future__ import print_function
+
+import json
 from collections import OrderedDict
 import warnings
 
-from six import iteritems, itervalues
+from six import iteritems, itervalues, string_types
 
 import numpy as np
 
@@ -761,8 +763,10 @@ def set_simul_deriv_color(self, simul_info):
 
         Parameters
         ----------
-        simul_info : ({dv1: colors, ...}, {resp1: {dv1: {0: [res_idxs, dv_idxs]} ...} ...})
-            Information about simultaneous coloring for design vars and responses.
+        simul_info : str or ({dv1: colors, ...}, {resp1: {dv1: {0: [res_idxs, dv_idxs]} ...} ...})
+            Information about simultaneous coloring for design vars and responses.  If a string,
+            then simul_info is assumed to be the name of a file that contains the coloring
+            information in JSON format.
         """
         if self.supports['simultaneous_derivatives']:
             self._simul_coloring_info = simul_info
@@ -785,6 +789,10 @@ def _setup_simul_coloring(self, mode='fwd'):
 
         prom2abs = self._problem.model._var_allprocs_prom2abs_list['output']
 
+        if isinstance(self._simul_coloring_info, string_types):
+            with open(self._simul_coloring_info, 'r') as f:
+                self._simul_coloring_info = json.load(f)
+
         coloring, maps = self._simul_coloring_info
         for dv, colors in iteritems(coloring):
             if dv not in self._designvars:
@@ -799,8 +807,9 @@ def _setup_simul_coloring(self, mode='fwd'):
             self._responses[res]['simul_map'] = dvdict
 
             for dv, col_dict in dvdict.items():
+                col_dict = {int(k): v for k, v in iteritems(col_dict)}
                 if dv not in self._designvars:
                     # convert name from promoted to absolute and replace dictionary key
                     del dvdict[dv]
                     dv = prom2abs[dv][0]
-                    dvdict[dv] = col_dict
+                dvdict[dv] = col_dict

openmdao/docs/_exts/embed_shell_cmd.py

@@ -92,9 +92,10 @@ class EmbedShellCmdDirective(Directive):
     has_content = False
 
     option_spec = {
-        'cmd': unchanged,
-        'dir': unchanged,
-        'show_cmd': unchanged
+        'cmd': unchanged,        # shell command to execute
+        'dir': unchanged,        # working dir
+        'show_cmd': unchanged,   # set this to make the shell command visible
+        'stderr': unchanged      # set this to include stderr contents with the output
     }
 
     def run(self):
@@ -115,10 +116,15 @@ def run(self):
         else:
             workdir = os.getcwd()
 
+        if 'stderr' in self.options:
+            stderr = subprocess.STDOUT
+        else:
+            stderr = None
+
         os.chdir(workdir)
 
         try:
-            output = subprocess.check_output(cmd, stderr=subprocess.STDOUT).decode('utf-8', 'ignore')
+            output = subprocess.check_output(cmd, stderr=stderr).decode('utf-8', 'ignore')
         except subprocess.CalledProcessError as err:
             raise SphinxError("Running of embedded shell command '{}' in docs failed. "
                               "Output was: \n{}".format(cmdstr, err.output.decode('utf-8')))

openmdao/docs/feature_reference/core_features/working_with_derivatives/simul_derivs.rst

@@ -155,43 +155,43 @@ would look like this:
     {
        'delta_theta_con.g': {
           'indeps.x': {
-             0: ([0, 1, 2, 3, 4], [0, 2, 4, 6, 8]),
-             1: ([0, 1, 2, 3, 4], [1, 3, 5, 7, 9]),
+             0: [[0, 1, 2, 3, 4], [0, 2, 4, 6, 8]],
+             1: [[0, 1, 2, 3, 4], [1, 3, 5, 7, 9]],
           },
           'indeps.y': {
-             0: ([0, 1, 2, 3, 4], [0, 2, 4, 6, 8]),
-             1: ([0, 1, 2, 3, 4], [1, 3, 5, 7, 9]),
+             0: [[0, 1, 2, 3, 4], [0, 2, 4, 6, 8]],
+             1: [[0, 1, 2, 3, 4], [1, 3, 5, 7, 9]],
+          },
+       },
+       'l_conx.g': {
+          'indeps.x': {
+             0: [[0], [0]],
           },
        },
        'r_con.g': {
           'indeps.x': {
-             0: ([0, 2, 4, 6, 8], [0, 2, 4, 6, 8]),
-             1: ([1, 3, 5, 7, 9], [1, 3, 5, 7, 9]),
+             0: [[0, 2, 4, 6, 8], [0, 2, 4, 6, 8]],
+             1: [[1, 3, 5, 7, 9], [1, 3, 5, 7, 9]],
           },
           'indeps.y': {
-             0: ([0, 2, 4, 6, 8], [0, 2, 4, 6, 8]),
-             1: ([1, 3, 5, 7, 9], [1, 3, 5, 7, 9]),
+             0: [[0, 2, 4, 6, 8], [0, 2, 4, 6, 8]],
+             1: [[1, 3, 5, 7, 9], [1, 3, 5, 7, 9]],
           },
        },
        'theta_con.g': {
           'indeps.x': {
-             0: ([0, 1, 2, 3, 4], [0, 2, 4, 6, 8]),
+             0: [[0, 1, 2, 3, 4], [0, 2, 4, 6, 8]],
           },
           'indeps.y': {
-             0: ([0, 1, 2, 3, 4], [0, 2, 4, 6, 8]),
-          },
-       },
-       'l_conx.g': {
-          'indeps.x': {
-             0: ([0], [0]),
+             0: [[0, 1, 2, 3, 4], [0, 2, 4, 6, 8]],
           },
        },
     })
 
     Coloring Summary
-    indeps.x num colors: 2
-    indeps.y num colors: 2
-    indeps.r num colors: 1
+    indeps.x num colors: 2   size: 10
+    indeps.y num colors: 2   size: 10
+    indeps.r num colors: 1   size: 1
     Total colors vs. total size: 5 vs 21
 
 
@@ -201,9 +201,38 @@ of performance improvement you should see when computing your total derivatives.
 the output show above, the total number of linear solves to compute the total jacobian will drop
 from 21 down to 5.
 
+It may be more convenient, especially for larger colorings, to use the `-o` command line option
+to output the coloring to a file as follows:
+
+.. code-block:: none
+
+    openmdao simul_coloring <your_script_name> -o my_coloring.json
+
+
+The coloring will be written in json format to the given file and can be loaded using the
+*set_simul_deriv_color* function like this:
+
+
+.. code-block:: python
+
+    prob.driver.set_simul_deriv_color('my_coloring.json')
+
+
 If you run *openmdao simul_coloring* and it turns out there is no simultaneous coloring available,
 don't be surprised.  Problems that have the necessary total jacobian sparsity to allow
-simultaneous derivatives are relatively uncommon.
+simultaneous derivatives are relatively uncommon.  If you think that your total jacobian is sparse
+enough that openmdao should be computing a smaller coloring than it gave you, then you can run
+the coloring algorigthm with a tolerance so that very small entries in the jacobian will be treated
+as zeros.  You can set this tolerance using the *-t* command line option as follows:
+
+
+.. code-block:: none
+
+    openmdao simul_coloring <your_script_name> -o my_coloring.json -t 1e-15
+
+
+Be careful when setting the tolerance however, because if you make it too large then you will be
+zeroing out jacobian entries that should not be ignored and your optimization may not converge.
 
 
 Checking that it works

openmdao/drivers/pyoptsparse_driver.py

@@ -556,17 +556,19 @@ def _setup_simul_coloring(self, mode='fwd'):
                 dv_dict = meta['simul_map']
                 self._res_jacs[res] = {}
                 for dv, col_dict in iteritems(dv_dict):
-                    rows = []
-                    cols = []
-                    for color, (row_idxs, col_idxs) in iteritems(col_dict):
-                        rows.append(row_idxs)
-                        cols.append(col_idxs)
-
-                    row = np.hstack(rows)
-                    col = np.hstack(cols)
-                    # print("sparsity for %s, %s: %d of %s" % (res, dv, row.size,
-                    #       (self._responses[res]['size'] * self._designvars[dv]['size'],)))
-                    self._res_jacs[res][dv] = {
-                        'coo': [row, col, np.zeros(row.size)],
-                        'shape': [self._responses[res]['size'], self._designvars[dv]['size']]
-                    }
+                    # don't set the sparsity unless the corresponding desvar coloring is set
+                    if self._designvars[dv]['simul_deriv_color']:
+                        rows = []
+                        cols = []
+                        for color, (row_idxs, col_idxs) in iteritems(col_dict):
+                            rows.append(row_idxs)
+                            cols.append(col_idxs)
+
+                        row = np.hstack(rows)
+                        col = np.hstack(cols)
+                        # print("sparsity for %s, %s: %d of %s" % (res, dv, row.size,
+                        #       (self._responses[res]['size'] * self._designvars[dv]['size'],)))
+                        self._res_jacs[res][dv] = {
+                            'coo': [row, col, np.zeros(row.size)],
+                            'shape': [self._responses[res]['size'], self._designvars[dv]['size']]
+                        }

openmdao/utils/array_utils.py

@@ -92,24 +92,56 @@ def convert_neg(arr, dim):
     return arr
 
 
-def array_viz(arr, stream=sys.stdout):
+def array_viz(arr, prob=None, of=None, wrt=None, stream=sys.stdout):
     """
-    Display the structure of an array in a compact form.
+    Display the structure of a boolean array in a compact form.
+
+    If prob, of, and wrt are supplied, print the name of the response alongside
+    each row and print the names of the design vars, aligned with each column, at
+    the bottom.
 
     Parameters
     ----------
     arr : ndarray
         Array being visualized.
+    prob : Problem or None
+        Problem object.
+    of : list of str or None
+        Names of response variables used in derivative calculation.
+    wrt : list of str or None
+        Names of design variables used in derivative calculation.
     stream : file-like
         Stream where output will be written.
     """
     if len(arr.shape) != 2:
         raise RuntimeError("array_viz only works for 2d arrays.")
 
-    for r in range(arr.shape[0]):
-        for c in range(arr.shape[1]):
-            if arr[r, c] == 0.0:
-                stream.write('.')
-            else:
-                stream.write('x')
-        stream.write(' %d\n' % r)
+    if prob is None or of is None or wrt is None:
+        for r in range(arr.shape[0]):
+            for c in range(arr.shape[1]):
+                if arr[r, c]:
+                    stream.write('x')
+                else:
+                    stream.write('.')
+            stream.write(' %d\n' % r)
+    else:
+
+        row = 0
+        for res in of:
+            for r in range(row, row + prob.driver._responses[res]['size']):
+                col = 0
+                for dv in wrt:
+                    for c in range(col, col + prob.driver._designvars[dv]['size']):
+                        if arr[r, c]:
+                            stream.write('x')
+                        else:
+                            stream.write('.')
+                    col = c + 1
+                stream.write(' %d  %s\n' % (r, res))
+            row = r + 1
+
+        start = 0
+        for name in wrt:
+            tab = ' ' * start
+            stream.write('%s|%s\n' % (tab, name))
+            start += prob.driver._designvars[name]['size']