Add parameter 'search_parents' to run_notebook/run_notebook_in_process

NotebookScripter/main.py

@@ -17,9 +17,8 @@
 __notebookscripter_injected__ = []
 
 
-def init_injected_parameters(**kwords):
-    next_injected = kwords
-    __notebookscripter_injected__.append(next_injected)
+def init_injected_parameters(injected_parameters, run_settings):
+    __notebookscripter_injected__.append([injected_parameters, run_settings])
 
 
 def deinit_injected_parameters():
@@ -35,18 +34,31 @@ def receive_parameter(**kwords):
     The default value is returned if the parameter was not provided in the
     call to run_notebook.
     """
-    module_namespace = __notebookscripter_injected__[-1] if __notebookscripter_injected__ else {}
+
+    # can't do this because kword argument order is not preserved in some python versions ...
+    if len(kwords) != 1:
+        raise ValueError("Exactly 1 kword argument must be passed to receive_parameter")
+
+    last_parameter_namespace, options_namespace = __notebookscripter_injected__[-1] if __notebookscripter_injected__ else [{}, {}]
+
+    if options_namespace.get("search_parents", None):
+        namespaces_to_search = [i for i, _ in __notebookscripter_injected__]
+    else:
+        namespaces_to_search = [last_parameter_namespace]
 
     ret = []
-    for (param_name, default_value) in kwords.items():
+
+    # search the namespaces in reverse order
+    param_name, default_value = next(iter(kwords.items()))
+    for module_namespace in reversed(namespaces_to_search):
         if param_name in module_namespace:
             ret.append(module_namespace[param_name])
-        else:
-            ret.append(default_value)
 
-    # can't do this because kword argument order is note preserved in some python versions ...
-    if len(kwords) > 1:
-        raise ValueError("Only 1 kword argument may be passed to receive_parameter")
+    # search space did not contain item -- use default value
+    if not ret:
+        ret.append(default_value)
+
+    # return the found item
     return ret[0]
 
 
@@ -105,10 +117,18 @@ def unregister_magics():
 def run_notebook(
         path_to_notebook: str,
         with_backend='agg',
+        search_parents: bool = False,
         **hooks
 ) -> typing.Any:
-    """Run a notebook as a module within this processes namespace"""
-
+    """Run a notebook within calling process
+
+    Args:
+        path_to_notebook: Path to .ipynb or .py file containing notebook code
+        with_backend: Override behavior of ipython's matplotlib 'magic directive' -- "% matplotlib inline" 
+        search_parents: receive_parameter() calls within the called notebook will search for parameters pass to any 'parent' invocations of run_notebook on the call stack, not just for parameters passed to this call
+    Returns:
+        Returns newly created (anonymous) python module in which the target code was executed.
+    """
     try:
         shell = NotebookScripterEmbeddedIpythonShell.instance()
     except MultipleInstanceError:
@@ -152,7 +172,9 @@ def matplotlib(self, _line):
     save_user_ns = shell.user_ns
     shell.user_ns = dynamic_module.__dict__
 
-    init_injected_parameters(**hooks)
+    init_injected_parameters(hooks, {
+        "search_parents": search_parents
+    })
 
     _, extension = os.path.splitext(path_to_notebook)
     if extension == ".ipynb":
@@ -185,8 +207,7 @@ def matplotlib(self, _line):
                     #     dynamic_module.__dict__.update(hooks.get(hook_name, {}))
         else:
             # execute .py files as notebooks
-            code = shell.input_transformer_manager.transform_cell(
-                file_source)
+            code = shell.input_transformer_manager.transform_cell(file_source)
 
             # run the code in the module, compile first to provide source mapping support
             code_block = compile(code, path_to_notebook, 'exec')
@@ -204,8 +225,8 @@ def matplotlib(self, _line):
     return dynamic_module
 
 
-def worker(queue, path_to_notebook, with_backend, return_values, **hooks):
-    dynamic_module = run_notebook(path_to_notebook, with_backend=with_backend, **hooks)
+def worker(queue, path_to_notebook, with_backend, search_parents, return_values, **hooks):
+    dynamic_module = run_notebook(path_to_notebook, with_backend=with_backend, search_parents=search_parents, **hooks)
 
     if return_values:
         ret = {k: simple_serialize(dynamic_module.__dict__[k]) for k in return_values if k in dynamic_module.__dict__}
@@ -223,16 +244,28 @@ def simple_serialize(obj):
 
 
 def run_notebook_in_process(
-        path_to_notebook: str,
-        with_backend='agg',
-        return_values=None,
-        **hooks
+    path_to_notebook: str,
+    with_backend='agg',
+    search_parents=False,
+    return_values=None,
+    **hooks
 ) -> None:
+    """Run a notebook in a new subprocess
+
+    Args:
+        path_to_notebook: Path to .ipynb or .py file containing notebook code
+        with_backend: Override behavior of ipython's matplotlib 'magic directive' -- "% matplotlib inline" 
+        search_parents: receive_parameter() calls within the called notebook will search for parameters pass to any 'parent' invocations of run_notebook on the call stack, not just for parameters passed to this call
+        return_values: Optional array of strings to pass back from subprocess -- values matching these names in the module created by invoking the notebook in a subprocess will be serialized passed across process boundaries back to this process, deserialized and made part of the returned module
+    Returns:
+        Returns newly created (anonymous) python module 
+        populated with requested values retrieved from the subprocess
+    """
     import multiprocessing as mp
 
     queue = mp.Queue()
 
-    p = mp.Process(target=worker, args=(queue, path_to_notebook, with_backend, return_values), kwargs=hooks)
+    p = mp.Process(target=worker, args=(queue, path_to_notebook, with_backend, search_parents, return_values), kwargs=hooks)
     p.start()
 
     module_identity = "loaded_notebook_from_subprocess"

README.md

@@ -77,6 +77,36 @@ When executed via run_notebook(..., with_backend='agg') - the line `%matplotlib
 
 This functionality supports 'interactive' plotting backend selection in the notebook environment and 'non-interactive' backend selection in the scripting context. 'agg' is a non-interactive backend built into most distributions of matplotlib. To disable this functionality provide `with_backend=None`.
 
+## Other options
+
+If desired - the parameter search_parents=True can be passed to run_notebook/run_notebook_in_process.
+
+Example:
+
+```python
+run_notebook("./parent.py", grandparent="grandparent")
+```
+
+parent.py
+
+```python
+from NotebookScripter import run_notebook
+run_notebook("child.py", search_parents=True)
+```
+
+child.py
+
+```python
+from NotebookScripter import receive_parameter
+param = receive_parameter(grandparent=None)
+
+print("Printed value will be "grandparent" rather than None: {0}".format(grandparent))
+```
+
+receive_parameter found the value for 'grandparent' passed to the ancestor call to run_notebook despite the fact that the call in parent.py passed no parameters.
+
+_Implementation Note_: Keyword parameters passed to run_notebook are stored in a stack. When search_parents is False, receive_parameter searches only the top frame of the parameters stack for matching variables. When search_parents is True then when a match isn't found on the top frame, parent frames are searched in order for matches with the default value returned when none of the stack's contain a matching value. The search_parents behavior depends only on the run_notebook() caller -- it is not inherited or itself influenced by any grandparent/child run_notebook invocations.
+
 ## Execute a notebook in isolated subprocess
 
 `run_notebook` executes notebook's within the same process as the caller. Sometimes more isolation between notebook executions is desired or required. NotebookScripter provides a run_notebook_in_process function for this case:
@@ -138,6 +168,10 @@ See [DEVELOPMENT_README.md](DEVELOPMENT_README.md)
 
 ## Changelog
 
+### 3.2.0
+
+- Add search_parents option to run_notebook and run_notebook_in_process (defaults to False)
+
 ### 3.1.3
 
 - Minor changes to boost test coverage %

setup.py

@@ -2,7 +2,7 @@
 
 setup(
     name='NotebookScripter',
-    version='3.1.3',
+    version='3.2.0',
     packages=('NotebookScripter',),
     url='https://github.com/breathe/NotebookScripter',
     license='MIT',

tests/SearchParents_1.pynotebook

@@ -0,0 +1,18 @@
+# notebook for testing search parents invocation behavior
+import os
+from NotebookScripter import run_notebook, receive_parameter
+
+my_a = receive_parameter(a="parent_a")
+
+notebook_file = os.path.join(os.path.dirname(__file__), "SearchParents_2.pynotebook")
+child_mod = run_notebook(notebook_file, b="parent_b")
+
+# test this
+without_search_parents = [my_a, child_mod.value]
+
+child_mod = run_notebook(notebook_file, search_parents=True, b="parent_b")
+
+# should be different
+with_search_parents = [my_a, child_mod.value]
+
+value = [without_search_parents, with_search_parents]

tests/SearchParents_2.pynotebook

@@ -0,0 +1,6 @@
+from NotebookScripter import receive_parameter
+
+a = receive_parameter(a="grandchild_a")
+b = receive_parameter(b="grandchild_b")
+
+value = [a, b]

tests/TestNotebookScripter.py

@@ -74,10 +74,11 @@ def put(self, item):
         queue = FakeQueue()
         notebook = self.notebook_file
         with_backend = "agg"
+        search_parents = False
         return_values = ["parameterized_name", "french_mode", "greeting_string", "hello"]
         hooks = {"parameterized_name": "external world"}
 
-        worker(queue, notebook, with_backend, return_values, **hooks)
+        worker(queue, notebook, with_backend, search_parents, return_values, **hooks)
         mod = queue._items[0]
 
         hello_repr = mod.pop("hello", None)
@@ -96,9 +97,26 @@ def setUp(self):
         self.testcase1_file = os.path.join(os.path.dirname(__file__), "./RecursiveSamples_1.pynotebook")
         self.testcase2_file = os.path.join(os.path.dirname(__file__), "./RecursiveSamples_2.pynotebook")
 
-    def test_run_recursive(self):
+    def test_run(self):
 
         for testcase in [self.testcase1_file, self.testcase2_file]:
             mod = NotebookScripter.run_notebook(self.notebook_file, parameter=testcase)
             value = mod.value
             self.assertMatchSnapshot(value)
+
+
+class TestSearchParents(snapshottest.TestCase):
+    """Test search_parents option"""
+
+    def setUp(self):
+        self.notebook_file = os.path.join(os.path.dirname(__file__), "./SearchParents_1.pynotebook")
+
+    def test_run_recursive(self):
+        mod = NotebookScripter.run_notebook(self.notebook_file, a="grandparent_a")
+        value = mod.value
+        self.assertMatchSnapshot(value)
+
+    def test_run_in_subprocess_recursive(self):
+        mod = NotebookScripter.run_notebook_in_process(self.notebook_file, return_values=["value"], a="grandparent_a")
+        value = mod.value
+        self.assertMatchSnapshot(value)

tests/snapshots/snap_TestNotebookScripter.py

@@ -61,20 +61,6 @@
     'parameterized_name': 'default world'
 }
 
-snapshots['TestNotebookExecution::test_run_notebook_with_hooks1 1'] = 'Hello external world'
-
-snapshots['TestNotebookExecution::test_run_notebook_with_hooks2 1'] = 'Salut external world2'
-
-snapshots['TestRecursiveNotebookExecution::test_run_recursive 1'] = 'Case 1 Expecting a string'
-
-snapshots['TestRecursiveNotebookExecution::test_run_recursive 2'] = 'Case 2 Expecting a string'
-
-snapshots['TestWorkerExecution::test_worker 1'] = {
-    'french_mode': None,
-    'greeting_string': 'Hello {0}',
-    'parameterized_name': 'external world'
-}
-
 snapshots['TestNotebookExecution::test_run_notebook_in_process_with_hooks 1'] = {
     '__doc__': None,
     '__loader__': None,
@@ -96,3 +82,51 @@
     'greeting_string': 'Salut {0}',
     'parameterized_name': 'external world2'
 }
+
+snapshots['TestNotebookExecution::test_run_notebook_with_hooks1 1'] = 'Hello external world'
+
+snapshots['TestNotebookExecution::test_run_notebook_with_hooks2 1'] = 'Salut external world2'
+
+snapshots['TestRecursiveNotebookExecution::test_run 1'] = 'Case 1 Expecting a string'
+
+snapshots['TestRecursiveNotebookExecution::test_run 2'] = 'Case 2 Expecting a string'
+
+snapshots['TestSearchParents::test_run_recursive 1'] = [
+    [
+        'grandparent_a',
+        [
+            'grandchild_a',
+            'parent_b'
+        ]
+    ],
+    [
+        'grandparent_a',
+        [
+            'grandparent_a',
+            'parent_b'
+        ]
+    ]
+]
+
+snapshots['TestWorkerExecution::test_worker 1'] = {
+    'french_mode': None,
+    'greeting_string': 'Hello {0}',
+    'parameterized_name': 'external world'
+}
+
+snapshots['TestSearchParents::test_run_in_subprocess_recursive 1'] = [
+    [
+        'grandparent_a',
+        [
+            'grandchild_a',
+            'parent_b'
+        ]
+    ],
+    [
+        'grandparent_a',
+        [
+            'grandparent_a',
+            'parent_b'
+        ]
+    ]
+]