Merge pull request #481 from impact27/magic_locals

PR: Allow magic to edit locals while debugging

spyder_kernels/customize/code_runner.py

@@ -41,7 +41,9 @@
 from spyder_kernels.customize.namespace_manager import NamespaceManager
 from spyder_kernels.customize.spyderpdb import SpyderPdb
 from spyder_kernels.customize.umr import UserModuleReloader
-from spyder_kernels.customize.utils import capture_last_Expr, canonic
+from spyder_kernels.customize.utils import (
+    capture_last_Expr, canonic, exec_encapsulate_locals
+)
 
 
 # For logging
@@ -496,11 +498,12 @@ def _exec_code(
 
             if capture_last_expression:
                 ast_code, capture_last_expression = capture_last_Expr(
-                    ast_code, "_spyder_out"
+                    ast_code, "_spyder_out", ns_globals
                 )
-                ns_globals["__spyder_builtins__"] = builtins
 
-            exec_fun(compile(ast_code, filename, "exec"), ns_globals, ns_locals)
+            exec_encapsulate_locals(
+                ast_code, ns_globals, ns_locals, exec_fun, filename
+            )
 
             if capture_last_expression:
                 out = ns_globals.pop("_spyder_out", None)

spyder_kernels/customize/spyderpdb.py

@@ -25,7 +25,9 @@
 
 import spyder_kernels
 from spyder_kernels.comms.frontendcomm import CommError, frontend_request
-from spyder_kernels.customize.utils import path_is_library, capture_last_Expr
+from spyder_kernels.customize.utils import (
+    path_is_library, capture_last_Expr, exec_encapsulate_locals
+)
 
 
 logger = logging.getLogger(__name__)
@@ -142,22 +144,22 @@ def default(self, line):
             if cmd == "debug":
                 return self.do_debug(arg)
 
-        locals = self.curframe_locals
-        globals = self.curframe.f_globals
+        local_ns = self.curframe_locals
+        global_ns = self.curframe.f_globals
 
         if self.pdb_use_exclamation_mark:
             # Find pdb commands executed without !
             cmd, arg, line = self.parseline(line)
             if cmd:
                 cmd_in_namespace = (
-                    cmd in globals
-                    or cmd in locals
+                    cmd in global_ns
+                    or cmd in local_ns
                     or cmd in builtins.__dict__
                 )
                 # Special case for quit and exit
                 if cmd in ("quit", "exit"):
-                    if cmd in globals and isinstance(
-                            globals[cmd], ZMQExitAutocall):
+                    if cmd in global_ns and isinstance(
+                            global_ns[cmd], ZMQExitAutocall):
                         # Use the pdb call
                         cmd_in_namespace = False
                 cmd_func = getattr(self, 'do_' + cmd, None)
@@ -181,6 +183,7 @@ def default(self, line):
                         # The pdb command is masked by something
                         self.print_exclamation_warning()
         try:
+            is_magic = line.startswith("%")
             line = TransformerManager().transform_cell(line)
             save_stdout = sys.stdout
             save_stdin = sys.stdin
@@ -199,82 +202,19 @@ def default(self, line):
                     capture_last_expression = False
                 else:
                     code_ast, capture_last_expression = capture_last_Expr(
-                        code_ast, "_spyderpdb_out")
-
-                globals["__spyder_builtins__"] = builtins
-
-                if locals is not globals:
-                    # Mitigates a behaviour of CPython that makes it difficult
-                    # to work with exec and the local namespace
-                    # See:
-                    #  - https://bugs.python.org/issue41918
-                    #  - https://bugs.python.org/issue46153
-                    #  - https://bugs.python.org/issue21161
-                    #  - spyder-ide/spyder#13909
-                    #  - spyder-ide/spyder-kernels#345
-                    #
-                    # The idea here is that the best way to emulate being in a
-                    # function is to actually execute the code in a function.
-                    # A function called `_spyderpdb_code` is created and
-                    # called. It will first load the locals, execute the code,
-                    # and then update the locals.
-                    #
-                    # One limitation of this approach is that locals() is only
-                    # a copy of the curframe locals. This means that closures
-                    # for example are early binding instead of late binding.
-
-                    # Create a function
-                    indent = "    "
-                    code = ["def _spyderpdb_code():"]
-
-                    # Add locals in globals
-                    # If the debugger is recursive, the globals could already
-                    # have a _spyderpdb_locals as it might be shared between
-                    # levels
-                    if "_spyderpdb_locals" in globals:
-                        globals["_spyderpdb_locals"].append(locals)
-                    else:
-                        globals["_spyderpdb_locals"] = [locals]
-
-                    # Load locals if they have a valid name
-                    # In comprehensions, locals could contain ".0" for example
-                    code += [indent + "{k} = _spyderpdb_locals[-1]['{k}']".format(
-                        k=k) for k in locals if k.isidentifier()]
-
-                    # The code comes here
-
-                    # Update the locals
-                    code += [indent + "_spyderpdb_locals[-1].update("
-                             "__spyder_builtins__.locals())"]
-
-                    # Run the function
-                    code += ["_spyderpdb_code()"]
-
-                    # Parse the function
-                    fun_ast = ast.parse('\n'.join(code) + '\n')
-
-                    # Inject code_ast in the function before the locals update
-                    fun_ast.body[0].body = (
-                        fun_ast.body[0].body[:-1]  # The locals
-                        + code_ast.body  # Code to run
-                        + fun_ast.body[0].body[-1:]  # Locals update
-                    )
-                    code_ast = fun_ast
-
-                try:
-                    exec(compile(code_ast, "<stdin>", "exec"), globals)
-                finally:
-                    if locals is not globals:
-                        # CLeanup code
-                        globals.pop("_spyderpdb_code", None)
-                        if len(globals["_spyderpdb_locals"]) > 1:
-                            del globals["_spyderpdb_locals"][-1]
-                        else:
-                            del globals["_spyderpdb_locals"]
-
+                        code_ast, "_spyderpdb_out", global_ns)
+
+                if is_magic:
+                    # Magics like runcell use and modify local_ns.
+                    # But the locals() dict can not be directly modified when
+                    # encapsulated. Therefore they must encapsulate the locals
+                    # themselves (see code_runner.py).
+                    exec(compile(code_ast, "<stdin>", "exec"), global_ns, local_ns)
+                else:
+                    exec_encapsulate_locals(code_ast, global_ns, local_ns)
 
                 if capture_last_expression:
-                    out = globals.pop("_spyderpdb_out", None)
+                    out = global_ns.pop("_spyderpdb_out", None)
                     if out is not None:
                         sys.stdout.flush()
                         sys.stderr.flush()
@@ -360,7 +300,7 @@ def stop_here(self, frame):
             # This is spyder-kernels internals
             return False
         return True
-    
+
     def should_continue(self, frame):
         """
         Jump to first breakpoint if needed.
@@ -617,9 +557,9 @@ def do_debug(self, arg):
         with self.recursive_debugger() as debugger:
             self.message("Entering recursive debugger")
             try:
-                globals = self.curframe.f_globals
-                locals = self.curframe_locals
-                return sys.call_tracing(debugger.run, (arg, globals, locals))
+                global_ns = self.curframe.f_globals
+                local_ns = self.curframe_locals
+                return sys.call_tracing(debugger.run, (arg, global_ns, local_ns))
             except Exception:
                 exc_info = sys.exc_info()[:2]
                 self.error(

spyder_kernels/customize/utils.py

@@ -6,6 +6,7 @@
 """Utility functions."""
 
 import ast
+import builtins
 import os
 import re
 import sys
@@ -96,18 +97,15 @@ def path_is_library(path, initial_pathlist=None):
         return False
 
 
-def capture_last_Expr(code_ast, out_varname):
-    """
-    Parse line and modify code to capture in globals the last expression.
-
-    The namespace must contain __spyder_builtins__, which is the builtins module.
-    """
+def capture_last_Expr(code_ast, out_varname, global_ns):
+    """Parse line and modify code to capture in globals the last expression."""
     # Modify ast code to capture the last expression
     capture_last_expression = False
     if (
         len(code_ast.body)
         and isinstance(code_ast.body[-1], ast.Expr)
     ):
+        global_ns["__spyder_builtins__"] = builtins
         capture_last_expression = True
         expr_node = code_ast.body[-1]
         # Create new assign node
@@ -131,6 +129,94 @@ def capture_last_Expr(code_ast, out_varname):
     return code_ast, capture_last_expression
 
 
+def exec_encapsulate_locals(
+    code_ast, globals, locals, exec_fun=None, filename=None
+):
+    """
+    Execute by encapsulating locals if needed.
+
+    Notes
+    ----- 
+    * In general, the dict returned by locals() might or might not be modified.
+      In this case, the encapsulated dict can not.
+    """
+    use_locals_hack = locals is not None and locals is not globals
+    if use_locals_hack:
+        globals["__spyder_builtins__"] = builtins
+
+        # Mitigates a behaviour of CPython that makes it difficult
+        # to work with exec and the local namespace
+        # See:
+        #  - https://bugs.python.org/issue41918
+        #  - https://bugs.python.org/issue46153
+        #  - https://bugs.python.org/issue21161
+        #  - spyder-ide/spyder#13909
+        #  - spyder-ide/spyder-kernels#345
+        #
+        # The idea here is that the best way to emulate being in a
+        # function is to actually execute the code in a function.
+        # A function called `_spyderpdb_code` is created and
+        # called. It will first load the locals, execute the code,
+        # and then update the locals.
+        #
+        # One limitation of this approach is that locals() is only
+        # a copy of the curframe locals. This means that closures
+        # for example are early binding instead of late binding.
+
+        # Create a function
+        indent = "    "
+        code = ["def _spyderpdb_code():"]
+
+        # Add locals in globals
+        # If the debugger is recursive, the globals could already
+        # have a _spyderpdb_locals as it might be shared between
+        # levels
+        if "_spyderpdb_locals" in globals:
+            globals["_spyderpdb_locals"].append(locals)
+        else:
+            globals["_spyderpdb_locals"] = [locals]
+
+        # Load locals if they have a valid name
+        # In comprehensions, locals could contain ".0" for example
+        code += [indent + "{k} = _spyderpdb_locals[-1]['{k}']".format(
+            k=k) for k in locals if k.isidentifier()]
+
+        # The code comes here
+
+        # Update the locals
+        code += [indent + "_spyderpdb_locals[-1].update("
+                 "__spyder_builtins__.locals())"]
+
+        # Run the function
+        code += ["_spyderpdb_code()"]
+
+        # Parse the function
+        fun_ast = ast.parse('\n'.join(code) + '\n')
+
+        # Inject code_ast in the function before the locals update
+        fun_ast.body[0].body = (
+            fun_ast.body[0].body[:-1]  # The locals
+            + code_ast.body  # Code to run
+            + fun_ast.body[0].body[-1:]  # Locals update
+        )
+        code_ast = fun_ast
+
+    try:
+        if exec_fun is None:
+            exec_fun = exec
+        if filename is None:
+            filename = "<stdin>"
+        exec_fun(compile(code_ast, filename, "exec"), globals)
+    finally:
+        if use_locals_hack:
+            # Cleanup code
+            globals.pop("_spyderpdb_code", None)
+            if len(globals["_spyderpdb_locals"]) > 1:
+                del globals["_spyderpdb_locals"][-1]
+            else:
+                del globals["_spyderpdb_locals"]
+
+
 def canonic(filename):
     """
     Return canonical form of filename.