Merge pull request ipython#648 from takluyver/usermod

Clean up handling of global namespaces with the proper semantics.

A global namespace should always be tied to a module: pickle accesses classes via the module in which they're defined. So I've changed the arguments for instantiating an InteractiveShell to include `user_module` in place of `user_global_ns`. The global namespace simply becomes a reference to `user_module.__dict__`.

For instantiating InteractiveShell, there are four possibilities:

* Neither `user_ns` nor `user_module` is given. A new (real) module is created named `__main__`, and its `__dict__` becomes the global and local namespace. This is what happens when starting IPython normally.
* Only `user_module` is given. Its `__dict__` becomes the global and local namespace.
* Both `user_ns` and `user_module` are given. `user_module.__dict__` is the global namespace, and `user_ns` is the local namespace. Note that we can't interactively define closures over variables in the local namespace (this seems to be a limitation of Python).
* Only `user_ns` is given. It is treated as the global and local namespace, and a `DummyMod` object is created to refer to it. This is intended as a convenience, especially for the test suite. The recommended way to pass in a single global namespace is as a reference to the module.

`embed()` digs out the locals and the module from the frame in which it's called.

Closes ipythongh-29, closes ipythongh-693.

IPython/core/displayhook.py

@@ -265,16 +265,16 @@ def update_user_ns(self, result):
                 self.___ = self.__
                 self.__ = self._
                 self._ = result
-                self.shell.user_ns.update({'_':self._,
-                                           '__':self.__,
-                                           '___':self.___})
+                self.shell.push({'_':self._,
+                                 '__':self.__,
+                                '___':self.___}, interactive=False)
 
             # hackish access to top-level  namespace to create _1,_2... dynamically
             to_main = {}
             if self.do_full_cache:
                 new_result = '_'+`self.prompt_count`
                 to_main[new_result] = result
-                self.shell.user_ns.update(to_main)
+                self.shell.push(to_main, interactive=False)
                 self.shell.user_ns['_oh'][self.prompt_count] = result
 
     def log_output(self, format_dict):

IPython/core/history.py

@@ -559,7 +559,8 @@ def store_inputs(self, line_num, source, source_raw=None):
                    '_ii': self._ii,
                    '_iii': self._iii,
                    new_i : self._i00 }
-        self.shell.user_ns.update(to_main)
+
+        self.shell.push(to_main, interactive=False)
 
     def store_output(self, line_num):
         """If database output logging is enabled, this saves all the

IPython/core/magic.py

@@ -748,11 +748,10 @@ def magic_who_ls(self, parameter_s=''):
         """
 
         user_ns = self.shell.user_ns
-        internal_ns = self.shell.internal_ns
         user_ns_hidden = self.shell.user_ns_hidden
         out = [ i for i in user_ns
                 if not i.startswith('_') \
-                and not (i in internal_ns or i in user_ns_hidden) ]
+                and not i in user_ns_hidden ]
 
         typelist = parameter_s.split()
         if typelist:

IPython/core/prefilter.py

@@ -86,7 +86,7 @@ def is_shadowed(identifier, ip):
     than ifun, because it can not contain a '.' character."""
     # This is much safer than calling ofind, which can change state
     return (identifier in ip.user_ns \
-            or identifier in ip.internal_ns \
+            or identifier in ip.user_global_ns \
             or identifier in ip.ns_table['builtin'])
 
 

IPython/core/tests/test_interactiveshell.py

@@ -25,6 +25,7 @@
 import tempfile
 import unittest
 from os.path import join
+import sys
 from StringIO import StringIO
 
 from IPython.testing import decorators as dec
@@ -128,6 +129,7 @@ def __repr__(self):
         f = IPython.core.formatters.PlainTextFormatter()
         f([Spam(),Spam()])
 
+
     def test_future_flags(self):
         """Check that future flags are used for parsing code (gh-777)"""
         ip = get_ipython()
@@ -151,6 +153,37 @@ def test_future_unicode(self):
         finally:
             # Reset compiler flags so we don't mess up other tests.
             ip.compile.reset_compiler_flags()
+
+    def test_can_pickle(self):
+        "Can we pickle objects defined interactively (GH-29)"
+        ip = get_ipython()
+        ip.reset()
+        ip.run_cell(("class Mylist(list):\n"
+                     "    def __init__(self,x=[]):\n"
+                     "        list.__init__(self,x)"))
+        ip.run_cell("w=Mylist([1,2,3])")
+
+        from cPickle import dumps
+
+        # We need to swap in our main module - this is only necessary
+        # inside the test framework, because IPython puts the interactive module
+        # in place (but the test framework undoes this).
+        _main = sys.modules['__main__']
+        sys.modules['__main__'] = ip.user_module
+        try:
+            res = dumps(ip.user_ns["w"])
+        finally:
+            sys.modules['__main__'] = _main
+        self.assertTrue(isinstance(res, bytes))
+
+    def test_global_ns(self):
+        "Code in functions must be able to access variables outside them."
+        ip = get_ipython()
+        ip.run_cell("a = 10")
+        ip.run_cell(("def f(x):\n"
+                     "    return x + a"))
+        ip.run_cell("b = f(12)")
+        self.assertEqual(ip.user_ns["b"], 22)
 
     def test_bad_custom_tb(self):
         """Check that InteractiveShell is protected from bad custom exception handlers"""

IPython/core/tests/test_iplib.py

@@ -28,19 +28,16 @@
 # Test functions
 #-----------------------------------------------------------------------------
 
-@dec.parametric
 def test_reset():
     """reset must clear most namespaces."""
-    # The number of variables in the private user_ns_hidden is not zero, but it
-    # should be constant regardless of what we do
-    nvars_config_ns = len(ip.user_ns_hidden)
 
     # Check that reset runs without error
     ip.reset()
 
     # Once we've reset it (to clear of any junk that might have been there from
     # other tests, we can count how many variables are in the user's namespace
     nvars_user_ns = len(ip.user_ns)
+    nvars_hidden = len(ip.user_ns_hidden)
 
     # Now add a few variables to user_ns, and check that reset clears them
     ip.user_ns['x'] = 1
@@ -49,15 +46,8 @@ def test_reset():
 
     # Finally, check that all namespaces have only as many variables as we
     # expect to find in them:
-    for ns in ip.ns_refs_table:
-        if ns is ip.user_ns:
-            nvars_expected = nvars_user_ns
-        elif ns is ip.user_ns_hidden:
-            nvars_expected = nvars_config_ns
-        else:
-            nvars_expected = 0
-
-        yield nt.assert_equals(len(ns), nvars_expected)
+    nt.assert_equals(len(ip.user_ns), nvars_user_ns)
+    nt.assert_equals(len(ip.user_ns_hidden), nvars_hidden)
 
 
 # Tests for reporting of exceptions in various modes, handling of SystemExit,

IPython/core/tests/test_run.py

@@ -228,3 +228,15 @@ def test_tclass(self):
         else:
             err = None
         tt.ipexec_validate(self.fname, out, err)
+
+    def test_run_i_after_reset(self):
+        """Check that %run -i still works after %reset (gh-693)"""
+        src = "yy = zz\n"
+        self.mktmp(src)
+        _ip.run_cell("zz = 23")
+        _ip.magic('run -i %s' % self.fname)
+        tt.assert_equals(_ip.user_ns['yy'], 23)
+        _ip.magic('reset -f')
+        _ip.run_cell("zz = 23")
+        _ip.magic('run -i %s' % self.fname)
+        tt.assert_equals(_ip.user_ns['yy'], 23)

IPython/frontend/terminal/embed.py

@@ -72,13 +72,13 @@ class InteractiveShellEmbed(TerminalInteractiveShell):
     display_banner = CBool(True)
 
     def __init__(self, config=None, ipython_dir=None, user_ns=None,
-                 user_global_ns=None, custom_exceptions=((),None),
+                 user_module=None, custom_exceptions=((),None),
                  usage=None, banner1=None, banner2=None,
                  display_banner=None, exit_msg=u''):
 
         super(InteractiveShellEmbed,self).__init__(
             config=config, ipython_dir=ipython_dir, user_ns=user_ns,
-            user_global_ns=user_global_ns, custom_exceptions=custom_exceptions,
+            user_module=user_module, custom_exceptions=custom_exceptions,
             usage=usage, banner1=banner1, banner2=banner2,
             display_banner=display_banner
         )
@@ -95,7 +95,7 @@ def __init__(self, config=None, ipython_dir=None, user_ns=None,
     def init_sys_modules(self):
         pass
 
-    def __call__(self, header='', local_ns=None, global_ns=None, dummy=None,
+    def __call__(self, header='', local_ns=None, module=None, dummy=None,
                  stack_depth=1):
         """Activate the interactive interpreter.
 
@@ -140,14 +140,14 @@ def __call__(self, header='', local_ns=None, global_ns=None, dummy=None,
 
         # Call the embedding code with a stack depth of 1 so it can skip over
         # our call and get the original caller's namespaces.
-        self.mainloop(local_ns, global_ns, stack_depth=stack_depth)
+        self.mainloop(local_ns, module, stack_depth=stack_depth)
 
         self.banner2 = self.old_banner2
 
         if self.exit_msg is not None:
             print self.exit_msg
 
-    def mainloop(self, local_ns=None, global_ns=None, stack_depth=0,
+    def mainloop(self, local_ns=None, module=None, stack_depth=0,
                  display_banner=None):
         """Embeds IPython into a running python program.
 
@@ -172,32 +172,37 @@ def mainloop(self, local_ns=None, global_ns=None, stack_depth=0,
         there is no fundamental reason why it can't work perfectly."""
 
         # Get locals and globals from caller
-        if local_ns is None or global_ns is None:
+        if local_ns is None or module is None:
             call_frame = sys._getframe(stack_depth).f_back
 
             if local_ns is None:
                 local_ns = call_frame.f_locals
-            if global_ns is None:
+            if module is None:
                 global_ns = call_frame.f_globals
-
+                module = sys.modules[global_ns['__name__']]
+
+        # Save original namespace and module so we can restore them after 
+        # embedding; otherwise the shell doesn't shut down correctly.
+        orig_user_module = self.user_module
+        orig_user_ns = self.user_ns
+
         # Update namespaces and fire up interpreter
-
+        
         # The global one is easy, we can just throw it in
-        self.user_global_ns = global_ns
+        self.user_module = module
 
-        # but the user/local one is tricky: ipython needs it to store internal
-        # data, but we also need the locals.  We'll copy locals in the user
-        # one, but will track what got copied so we can delete them at exit.
-        # This is so that a later embedded call doesn't see locals from a
-        # previous call (which most likely existed in a separate scope).
-        local_varnames = local_ns.keys()
-        self.user_ns.update(local_ns)
-        #self.user_ns['local_ns'] = local_ns  # dbg
+        # But the user/local one is tricky: ipython needs it to store internal
+        # data, but we also need the locals. We'll throw our hidden variables
+        # like _ih and get_ipython() into the local namespace, but delete them
+        # later.
+        self.user_ns = local_ns
+        self.init_user_ns()
 
         # Patch for global embedding to make sure that things don't overwrite
         # user globals accidentally. Thanks to Richard <rxe@renre-europe.com>
         # FIXME. Test this a bit more carefully (the if.. is new)
-        if local_ns is None and global_ns is None:
+        # N.B. This can't now ever be called. Not sure what it was for.
+        if local_ns is None and module is None:
             self.user_global_ns.update(__main__.__dict__)
 
         # make sure the tab-completer has the correct frame information, so it
@@ -206,13 +211,14 @@ def mainloop(self, local_ns=None, global_ns=None, stack_depth=0,
 
         with nested(self.builtin_trap, self.display_trap):
             self.interact(display_banner=display_banner)
-
-            # now, purge out the user namespace from anything we might have added
-            # from the caller's local namespace
-            delvar = self.user_ns.pop
-            for var in local_varnames:
-                delvar(var,None)
-
+
+        # now, purge out the local namespace of IPython's hidden variables.
+        for name in self.user_ns_hidden:
+            local_ns.pop(name, None)
+
+        # Restore original namespace so shell can shut down when we exit.
+        self.user_module = orig_user_module
+        self.user_ns = orig_user_ns
 
 _embedded_shell = None
 

IPython/frontend/terminal/interactiveshell.py

@@ -172,13 +172,13 @@ class TerminalInteractiveShell(InteractiveShell):
     )
 
     def __init__(self, config=None, ipython_dir=None, profile_dir=None, user_ns=None,
-                 user_global_ns=None, custom_exceptions=((),None),
+                 user_module=None, custom_exceptions=((),None),
                  usage=None, banner1=None, banner2=None,
                  display_banner=None):
 
         super(TerminalInteractiveShell, self).__init__(
             config=config, profile_dir=profile_dir, user_ns=user_ns,
-            user_global_ns=user_global_ns, custom_exceptions=custom_exceptions
+            user_module=user_module, custom_exceptions=custom_exceptions
         )
         # use os.system instead of utils.process.system by default,
         # because piped system doesn't make sense in the Terminal:

IPython/testing/globalipapp.py

@@ -190,7 +190,6 @@ def start_ipython():
     # Create and initialize our test-friendly IPython instance.
     shell = TerminalInteractiveShell.instance(config=config,
                                               user_ns=ipnsdict(),
-                                              user_global_ns={}
                                               )
 
     # A few more tweaks needed for playing nicely with doctests...
@@ -206,8 +205,7 @@ def start_ipython():
     # can capture subcommands and print them to Python's stdout, otherwise the
     # doctest machinery would miss them.
     shell.system = py3compat.MethodType(xsys, shell)
-
-
+
     shell._showtraceback = py3compat.MethodType(_showtraceback, shell)
 
     # IPython is ready, now clean up some global state...

IPython/testing/plugin/ipdoctest.py

@@ -271,6 +271,8 @@ def setUp(self):
             # for IPython examples *only*, we swap the globals with the ipython
             # namespace, after updating it with the globals (which doctest
             # fills with the necessary info from the module being tested).
+            self.user_ns_orig = {}
+            self.user_ns_orig.update(_ip.user_ns)
             _ip.user_ns.update(self._dt_test.globs)
             self._dt_test.globs = _ip.user_ns
             # IPython must protect the _ key in the namespace (it can't exist)
@@ -286,6 +288,8 @@ def tearDown(self):
         # teardown doesn't destroy the ipython namespace
         if isinstance(self._dt_test.examples[0],IPExample):
             self._dt_test.globs = self._dt_test_globs_ori
+            _ip.user_ns.clear()
+            _ip.user_ns.update(self.user_ns_orig)
             # Restore the behavior of the '_' key in the user namespace to
             # normal after each doctest, so that unittests behave normally
             _ip.user_ns.protect_underscore = False

docs/source/whatsnew/development.txt

@@ -97,6 +97,10 @@ Major Bugs fixed
 * IPython no longer crashes when started on recent versions of Python 3 in
   Windows (:ghissue:`737`).
 
+* Instances of classes defined interactively can now be pickled (:ghissue:`29`;
+  :ghpull:`648`). Note that pickling saves a reference to the class definition,
+  so unpickling the instances will only work where the class has been defined.
+
 .. * use bullet list
 
 Backwards incompatible changes
@@ -132,4 +136,10 @@ Backwards incompatible changes
   The full path will still work, and is necessary for using custom launchers not in
   IPython's launcher module.
 
+* For embedding a shell, note that the parameter ``user_global_ns`` has been
+  replaced by ``user_module``, and expects a module-like object, rather than
+  a namespace dict. The ``user_ns`` parameter works the same way as before, and
+  calling :func:`~IPython.frontend.terminal.embed.embed` with no arguments still
+  works the same way.
+
 .. * use bullet list