Merge pull request #1844 from DKilkenny/notebooks

Added detection of notebook env, table formatting and in cell HTML output

openmdao/api.py

@@ -8,6 +8,7 @@
 from openmdao.core.implicitcomponent import ImplicitComponent
 from openmdao.core.indepvarcomp import IndepVarComp
 from openmdao.core.analysis_error import AnalysisError
+from openmdao.core.notebook_mode import notebook_mode
 
 # Components
 from openmdao.components.add_subtract_comp import AddSubtractComp

openmdao/core/notebook_mode.py

@@ -0,0 +1,32 @@
+"""Checking for interactive notebook mode."""
+from openmdao.utils.general_utils import simple_warning
+
+try:
+    from tabulate import tabulate
+except ImportError:
+    tabulate = None
+
+
+def notebook_mode():
+    """
+    Check if the environment is interactive and if tabulate is installed.
+
+    Returns
+    -------
+    bool
+        True if the environment is an interactive notebook.
+    """
+    ipy = False
+    try:
+        from IPython import get_ipython
+        ipy = get_ipython() is not None
+    except ImportError:
+        pass
+
+    if ipy and tabulate is None:
+        simple_warning("Tabulate is not installed run `pip install openmdao[notebooks]` to "
+                       "install required dependencies. Using ASCII for outputs.")
+    return ipy
+
+
+notebook = notebook_mode()

openmdao/core/system.py

@@ -18,6 +18,7 @@
 import networkx as nx
 
 import openmdao
+from openmdao.core.notebook_mode import notebook, tabulate
 from openmdao.core.configinfo import _ConfigInfo
 from openmdao.core.constants import _DEFAULT_OUT_STREAM, _UNDEFINED, INT_DTYPE
 from openmdao.jacobians.assembled_jacobian import DenseJacobian, CSCJacobian
@@ -3478,7 +3479,18 @@ def list_inputs(self,
             out_stream = sys.stdout
 
         if out_stream:
-            self._write_table('input', inputs, hierarchical, print_arrays, all_procs, out_stream)
+            if notebook and tabulate is not None:
+                nb_format = {"Inputs": [], "value": [], "units": [], "shape": [],
+                             "global_shape": []}
+                for output, attrs in inputs.items():
+                    nb_format["Inputs"].append(output)
+                    for key, val in attrs.items():
+                        nb_format[key].append(val)
+
+                return tabulate(nb_format, headers="keys", tablefmt='html')
+            else:
+                self._write_table('input', inputs, hierarchical, print_arrays, all_procs,
+                                  out_stream)
 
         if self.pathname:
             # convert to relative names
@@ -3618,12 +3630,22 @@ def list_outputs(self,
         rel_idx = len(self.pathname) + 1 if self.pathname else 0
 
         states = set(self._list_states())
-
         if explicit:
             expl_outputs = {n: m for n, m in outputs.items() if n not in states}
             if out_stream:
-                self._write_table('explicit', expl_outputs, hierarchical, print_arrays,
-                                  all_procs, out_stream)
+                if notebook and tabulate is not None:
+                    nb_format = {"Explicit Output": [], "value": [], "units": [], "shape": [],
+                                 "global_shape": []}
+                    for output, attrs in expl_outputs.items():
+                        nb_format["Explicit Output"].append(output)
+                        for key, val in attrs.items():
+                            nb_format[key].append(val)
+
+                    return tabulate(nb_format, headers="keys", tablefmt='html')
+                else:
+                    self._write_table('explicit', expl_outputs, hierarchical, print_arrays,
+                                      all_procs, out_stream)
+
             if self.name:  # convert to relative name
                 expl_outputs = [(n[rel_idx:], meta) for n, meta in expl_outputs.items()]
             else:
@@ -3644,8 +3666,18 @@ def list_outputs(self,
             else:
                 impl_outputs = {n: m for n, m in outputs.items() if n in states}
             if out_stream:
-                self._write_table('implicit', impl_outputs, hierarchical, print_arrays,
-                                  all_procs, out_stream)
+                if notebook and tabulate is not None:
+                    nb_format = {"Implicit Output": [], "value": [], "units": [], "shape": [],
+                                 "global_shape": []}
+                    for output, attrs in expl_outputs.items():
+                        nb_format["Implicit Output"].append(output)
+                        for key, val in attrs.items():
+                            nb_format[key].append(val)
+
+                    return tabulate(nb_format, headers="keys", tablefmt='html')
+                else:
+                    self._write_table('implicit', impl_outputs, hierarchical, print_arrays,
+                                      all_procs, out_stream)
             if self.name:  # convert to relative name
                 impl_outputs = [(n[rel_idx:], meta) for n, meta in impl_outputs.items()]
             else:

openmdao/recorders/sqlite_reader.py

@@ -4,14 +4,12 @@
 import sqlite3
 from collections import OrderedDict
 
-from io import StringIO
-
 import sys
 import numpy as np
 
 from openmdao.recorders.base_case_reader import BaseCaseReader
 from openmdao.recorders.case import Case
-
+from openmdao.core.notebook_mode import notebook, tabulate
 from openmdao.core.constants import _DEFAULT_OUT_STREAM
 from openmdao.utils.general_utils import simple_warning
 from openmdao.utils.variable_table import write_source_table
@@ -342,10 +340,11 @@ def list_sources(self, out_stream=_DEFAULT_OUT_STREAM):
         if self._format_version >= 2 and self._problem_cases.count() > 0:
             sources.extend(self._problem_cases.list_sources())
 
-        if out_stream:
+        if notebook and tabulate is not None:
+            return tabulate([sources], headers=["Source"], tablefmt='html')
+        elif out_stream:
             if out_stream is _DEFAULT_OUT_STREAM:
                 out_stream = sys.stdout
-
             for source in sources:
                 out_stream.write('{}\n'.format(source))
 
@@ -607,7 +606,11 @@ def list_cases(self, source=None, recurse=True, flat=True, out_stream=_DEFAULT_O
                             (source, type(source).__name__))
 
         if not source:
-            return self._list_cases_recurse_flat(out_stream=out_stream)
+            if notebook and tabulate is not None:
+                cases = self._list_cases_recurse_flat(out_stream=out_stream)
+                return tabulate(cases, headers="keys", tablefmt='html')
+            else:
+                return self._list_cases_recurse_flat(out_stream=out_stream)
 
         elif source == 'problem':
             if self._format_version >= 2:
@@ -628,7 +631,10 @@ def list_cases(self, source=None, recurse=True, flat=True, out_stream=_DEFAULT_O
                 case_table = None
 
             if case_table is not None:
-                if not recurse:
+                if notebook and tabulate is not None:
+                    cases = [[case] for case in case_table._cases.keys()]
+                    return tabulate(cases, headers=[source], tablefmt='html')
+                elif not recurse:
                     # return list of cases from the source alone
                     return case_table.list_cases(source)
                 elif flat:
@@ -722,7 +728,11 @@ def _list_cases_recurse_flat(self, coord=None, out_stream=_DEFAULT_OUT_STREAM):
             if out_stream is _DEFAULT_OUT_STREAM:
                 out_stream = sys.stdout
 
-            write_source_table(self.source_cases_table, out_stream)
+            if notebook:
+                nb_format = {key: [val] for key, val in self.source_cases_table.items() if val}
+                return nb_format
+            else:
+                write_source_table(self.source_cases_table, out_stream)
 
         return cases
 

openmdao/recorders/tests/test_sqlite_reader.py

@@ -11,19 +11,21 @@
 import numpy as np
 from io import StringIO
 
-
 import openmdao.api as om
+import openmdao
 from openmdao.recorders.sqlite_recorder import format_version
 from openmdao.recorders.sqlite_reader import SqliteCaseReader
 from openmdao.recorders.tests.test_sqlite_recorder import ParaboloidProblem
 from openmdao.recorders.case import PromAbsDict
+from openmdao.core.notebook_mode import notebook_mode, tabulate
 from openmdao.core.tests.test_units import SpeedComp
 from openmdao.test_suite.components.expl_comp_array import TestExplCompArray
 from openmdao.test_suite.components.implicit_newton_linesearch import ImplCompTwoStates
 from openmdao.test_suite.components.paraboloid import Paraboloid
 from openmdao.test_suite.components.paraboloid_problem import ParaboloidProblem
 from openmdao.test_suite.components.sellar import SellarDerivativesGrouped, \
     SellarDis1withDerivatives, SellarDis2withDerivatives, SellarProblem
+from openmdao.test_suite.components.double_sellar import DoubleSellar
 from openmdao.utils.assert_utils import assert_near_equal, assert_warning
 from openmdao.utils.general_utils import set_pyoptsparse_opt, determine_adder_scaler, printoptions
 from openmdao.utils.general_utils import remove_whitespace
@@ -3154,6 +3156,162 @@ def test_list_source_vars_format(self):
         for i, line in enumerate(expected_cases):
             self.assertEqual(text[i], line)
 
+@unittest.skipUnless(tabulate, "Tabulate is required")
+class TestNotebookFormat(unittest.TestCase):
+
+    def setUp(self):
+        openmdao.core.system.notebook = True
+        openmdao.recorders.sqlite_reader.notebook = True
+
+        self.filename = "sqlite_test"
+        self.recorder = om.SqliteRecorder(self.filename, record_viewer_data=False)
+
+    def tearDown(self):
+        openmdao.core.system.notebook = False
+        openmdao.recorders.sqlite_reader.notebook = False
+
+    def test_list_inputs_and_outputs_notebook_format(self):
+
+        prob = om.Problem()
+        model = prob.model = DoubleSellar()
+
+        driver = prob.driver
+
+        recorder = om.SqliteRecorder("cases.sql")
+        prob.model.add_recorder(recorder)
+
+        driver.recording_options['record_desvars'] = False
+        driver.recording_options['record_objectives'] = False
+        driver.recording_options['record_constraints'] = False
+        driver.recording_options['record_derivatives'] = False
+        driver.add_recorder(recorder)
+
+        # each SubSellar group converges itself
+        g1 = model.g1
+        g1.nonlinear_solver = om.NewtonSolver(solve_subsystems=True)
+        g1.linear_solver = om.DirectSolver()  # used for derivatives
+
+        g2 = model.g2
+        g2.nonlinear_solver = om.NewtonSolver(solve_subsystems=True)
+        g2.linear_solver = om.DirectSolver()
+
+        # Converge the outer loop with Gauss Seidel, with a looser tolerance.
+        model.nonlinear_solver = om.NonlinearBlockGS(rtol=1.0e-5)
+        model.linear_solver = om.ScipyKrylov()
+        model.linear_solver.precon = om.LinearBlockGS()
+
+        prob.setup()
+        prob.run_model()
+
+        inputs = prob.model.list_inputs()
+
+        self.assertTrue("g1.d1.z" in inputs)
+        self.assertTrue("[0. 0.]" in inputs)
+        self.assertTrue("g1.d1.y2" in inputs)
+        self.assertTrue("[0.80000249]" in inputs)
+        self.assertTrue("Inputs" in inputs)
+        self.assertTrue("value" in inputs)
+        self.assertTrue("units" in inputs)
+        self.assertTrue("shape" in inputs)
+        self.assertTrue("global_shape" in inputs)
+
+        outputs = prob.model.list_outputs(units=True)
+
+        self.assertTrue("g1.d1.y1" in outputs)
+        self.assertTrue("0.640004" in outputs)
+        self.assertTrue("g1.d2.y2" in outputs)
+        self.assertTrue("0.800002" in outputs)
+        self.assertTrue("Explicit Output" in outputs)
+        self.assertTrue("value" in outputs)
+        self.assertTrue("units" in outputs)
+        self.assertTrue("shape" in outputs)
+        self.assertTrue("global_shape" in outputs)
+
+    def test_list_cases_format(self):
+
+        recorder = om.SqliteRecorder(self.filename, record_viewer_data=False)
+
+        prob = SellarProblem()
+        prob.setup()
+
+        prob.add_recorder(recorder)
+        prob.driver.add_recorder(recorder)
+        prob.model.d1.add_recorder(recorder)
+
+        prob.run_driver()
+
+        prob.record('final')
+        prob.cleanup()
+
+        cr = om.CaseReader(self.filename)
+
+        expected_cases = [
+            'system',
+            '    rank0:Driver|0|root._solve_nonlinear|0|d1._solve_nonlinear|0',
+            '    rank0:Driver|0|root._solve_nonlinear|0|NonlinearBlockGS|1|d1._solve_nonlinear|1',
+            '    rank0:Driver|0|root._solve_nonlinear|0|NonlinearBlockGS|2|d1._solve_nonlinear|2',
+            '    rank0:Driver|0|root._solve_nonlinear|0|NonlinearBlockGS|3|d1._solve_nonlinear|3',
+            '    rank0:Driver|0|root._solve_nonlinear|0|NonlinearBlockGS|4|d1._solve_nonlinear|4',
+            '    rank0:Driver|0|root._solve_nonlinear|0|NonlinearBlockGS|5|d1._solve_nonlinear|5',
+            '    rank0:Driver|0|root._solve_nonlinear|0|NonlinearBlockGS|6|d1._solve_nonlinear|6',
+            '    rank0:Driver|0|root._solve_nonlinear|0|NonlinearBlockGS|7|d1._solve_nonlinear|7',
+            'driver',
+            '    rank0:Driver|0',
+            'problem',
+            '    final',
+        ]
+
+        cases = cr.list_cases()
+
+        expected_case_outputs = [
+            'system',
+            'driver',
+            'problem',
+            'rank0:Driver|0|root._solve_nonlinear|0|d1._solve_nonlinear|0',
+            'rank0:Driver|0|root._solve_nonlinear|0|NonlinearBlockGS|1|d1._solve_nonlinear|1',
+            'rank0:Driver|0',
+            'final']
+
+        for i in expected_case_outputs:
+            self.assertTrue(i in cases)
+
+    def test_driver_source_list_cases(self):
+        prob = om.Problem()
+        model = prob.model
+
+        model.add_subsystem('p1', om.IndepVarComp('x', 0.0), promotes=['x'])
+        model.add_subsystem('p2', om.IndepVarComp('y', 0.0), promotes=['y'])
+        model.add_subsystem('comp', Paraboloid(), promotes=['x', 'y', 'f_xy'])
+
+        model.add_design_var('x', lower=0.0, upper=1.0)
+        model.add_design_var('y', lower=0.0, upper=1.0)
+        model.add_objective('f_xy')
+
+        prob.setup()
+
+        # create a list of DOE cases
+        case_gen = om.FullFactorialGenerator(levels=3)
+        cases = list(case_gen(model.get_design_vars(recurse=True)))
+
+        # create DOEDriver using provided list of cases
+        prob.driver = om.DOEDriver(cases)
+        prob.driver.add_recorder(om.SqliteRecorder("cases.sql"))
+
+        prob.run_driver()
+        prob.cleanup()
+
+        cr = om.CaseReader("cases.sql")
+        cases = cr.list_cases('driver')
+
+        expected_output = [
+            'driver',
+            'rank0:DOEDriver_List|0',
+            'rank0:DOEDriver_List|1'
+        ]
+
+        for i in expected_output:
+            self.assertTrue(i in cases)
+
 @use_tempdirs
 class TestFeatureSqliteReader(unittest.TestCase):
 

openmdao/visualization/connection_viewer/viewconns.py

@@ -8,7 +8,11 @@
 
 import numpy as np
 
+from IPython.display import IFrame, display
+
+import openmdao
 from openmdao.core.problem import Problem
+from openmdao.core.notebook_mode import notebook
 from openmdao.utils.units import convert_units
 from openmdao.utils.mpi import MPI
 from openmdao.utils.webview import webview
@@ -188,5 +192,8 @@ def view_connections(root, outfile='connections.html', show_browser=True,
         s = s.replace("<tabulator_style>", tabulator_style)
         f.write(s)
 
-    if show_browser:
+    if notebook:
+        display(IFrame(src=outfile, width=1000, height=1000))
+
+    if show_browser and not notebook:
         webview(outfile)