Permalink
Browse files

Merge pull request #648 from takluyver/usermod

Clean up handling of global namespaces with the proper semantics.

A global namespace should always be tied to a module: pickle accesses classes via the module in which they're defined. So I've changed the arguments for instantiating an InteractiveShell to include `user_module` in place of `user_global_ns`. The global namespace simply becomes a reference to `user_module.__dict__`.

For instantiating InteractiveShell, there are four possibilities:

* Neither `user_ns` nor `user_module` is given. A new (real) module is created named `__main__`, and its `__dict__` becomes the global and local namespace. This is what happens when starting IPython normally.
* Only `user_module` is given. Its `__dict__` becomes the global and local namespace.
* Both `user_ns` and `user_module` are given. `user_module.__dict__` is the global namespace, and `user_ns` is the local namespace. Note that we can't interactively define closures over variables in the local namespace (this seems to be a limitation of Python).
* Only `user_ns` is given. It is treated as the global and local namespace, and a `DummyMod` object is created to refer to it. This is intended as a convenience, especially for the test suite. The recommended way to pass in a single global namespace is as a reference to the module.

`embed()` digs out the locals and the module from the frame in which it's called.

Closes gh-29, closes gh-693.
  • Loading branch information...
2 parents 668e8a0 + b26bf6a commit a1e4911b6e7f973e3d5cf353e60473ab85be9b57 @fperez fperez committed Nov 27, 2011
@@ -265,16 +265,16 @@ def update_user_ns(self, result):
self.___ = self.__
self.__ = self._
self._ = result
- self.shell.user_ns.update({'_':self._,
- '__':self.__,
- '___':self.___})
+ self.shell.push({'_':self._,
+ '__':self.__,
+ '___':self.___}, interactive=False)
# hackish access to top-level namespace to create _1,_2... dynamically
to_main = {}
if self.do_full_cache:
new_result = '_'+`self.prompt_count`
to_main[new_result] = result
- self.shell.user_ns.update(to_main)
+ self.shell.push(to_main, interactive=False)
self.shell.user_ns['_oh'][self.prompt_count] = result
def log_output(self, format_dict):
View
@@ -559,7 +559,8 @@ def store_inputs(self, line_num, source, source_raw=None):
'_ii': self._ii,
'_iii': self._iii,
new_i : self._i00 }
- self.shell.user_ns.update(to_main)
+
+ self.shell.push(to_main, interactive=False)
def store_output(self, line_num):
"""If database output logging is enabled, this saves all the

Large diffs are not rendered by default.

Oops, something went wrong.
View
@@ -748,11 +748,10 @@ def magic_who_ls(self, parameter_s=''):
"""
user_ns = self.shell.user_ns
- internal_ns = self.shell.internal_ns
user_ns_hidden = self.shell.user_ns_hidden
out = [ i for i in user_ns
if not i.startswith('_') \
- and not (i in internal_ns or i in user_ns_hidden) ]
+ and not i in user_ns_hidden ]
typelist = parameter_s.split()
if typelist:
@@ -86,7 +86,7 @@ def is_shadowed(identifier, ip):
than ifun, because it can not contain a '.' character."""
# This is much safer than calling ofind, which can change state
return (identifier in ip.user_ns \
- or identifier in ip.internal_ns \
+ or identifier in ip.user_global_ns \
or identifier in ip.ns_table['builtin'])
@@ -25,6 +25,7 @@
import tempfile
import unittest
from os.path import join
+import sys
from StringIO import StringIO
from IPython.testing import decorators as dec
@@ -128,6 +129,7 @@ def __repr__(self):
f = IPython.core.formatters.PlainTextFormatter()
f([Spam(),Spam()])
+
def test_future_flags(self):
"""Check that future flags are used for parsing code (gh-777)"""
ip = get_ipython()
@@ -151,6 +153,37 @@ def test_future_unicode(self):
finally:
# Reset compiler flags so we don't mess up other tests.
ip.compile.reset_compiler_flags()
+
+ def test_can_pickle(self):
+ "Can we pickle objects defined interactively (GH-29)"
+ ip = get_ipython()
+ ip.reset()
+ ip.run_cell(("class Mylist(list):\n"
+ " def __init__(self,x=[]):\n"
+ " list.__init__(self,x)"))
+ ip.run_cell("w=Mylist([1,2,3])")
+
+ from cPickle import dumps
+
+ # We need to swap in our main module - this is only necessary
+ # inside the test framework, because IPython puts the interactive module
+ # in place (but the test framework undoes this).
+ _main = sys.modules['__main__']
+ sys.modules['__main__'] = ip.user_module
+ try:
+ res = dumps(ip.user_ns["w"])
+ finally:
+ sys.modules['__main__'] = _main
+ self.assertTrue(isinstance(res, bytes))
+
+ def test_global_ns(self):
+ "Code in functions must be able to access variables outside them."
+ ip = get_ipython()
+ ip.run_cell("a = 10")
+ ip.run_cell(("def f(x):\n"
+ " return x + a"))
+ ip.run_cell("b = f(12)")
+ self.assertEqual(ip.user_ns["b"], 22)
def test_bad_custom_tb(self):
"""Check that InteractiveShell is protected from bad custom exception handlers"""
@@ -28,19 +28,16 @@
# Test functions
#-----------------------------------------------------------------------------
-@dec.parametric
def test_reset():
"""reset must clear most namespaces."""
- # The number of variables in the private user_ns_hidden is not zero, but it
- # should be constant regardless of what we do
- nvars_config_ns = len(ip.user_ns_hidden)
# Check that reset runs without error
ip.reset()
# Once we've reset it (to clear of any junk that might have been there from
# other tests, we can count how many variables are in the user's namespace
nvars_user_ns = len(ip.user_ns)
+ nvars_hidden = len(ip.user_ns_hidden)
# Now add a few variables to user_ns, and check that reset clears them
ip.user_ns['x'] = 1
@@ -49,15 +46,8 @@ def test_reset():
# Finally, check that all namespaces have only as many variables as we
# expect to find in them:
- for ns in ip.ns_refs_table:
- if ns is ip.user_ns:
- nvars_expected = nvars_user_ns
- elif ns is ip.user_ns_hidden:
- nvars_expected = nvars_config_ns
- else:
- nvars_expected = 0
-
- yield nt.assert_equals(len(ns), nvars_expected)
+ nt.assert_equals(len(ip.user_ns), nvars_user_ns)
+ nt.assert_equals(len(ip.user_ns_hidden), nvars_hidden)
# Tests for reporting of exceptions in various modes, handling of SystemExit,
@@ -228,3 +228,15 @@ def test_tclass(self):
else:
err = None
tt.ipexec_validate(self.fname, out, err)
+
+ def test_run_i_after_reset(self):
+ """Check that %run -i still works after %reset (gh-693)"""
+ src = "yy = zz\n"
+ self.mktmp(src)
+ _ip.run_cell("zz = 23")
+ _ip.magic('run -i %s' % self.fname)
+ tt.assert_equals(_ip.user_ns['yy'], 23)
+ _ip.magic('reset -f')
+ _ip.run_cell("zz = 23")
+ _ip.magic('run -i %s' % self.fname)
+ tt.assert_equals(_ip.user_ns['yy'], 23)
@@ -72,13 +72,13 @@ class InteractiveShellEmbed(TerminalInteractiveShell):
display_banner = CBool(True)
def __init__(self, config=None, ipython_dir=None, user_ns=None,
- user_global_ns=None, custom_exceptions=((),None),
+ user_module=None, custom_exceptions=((),None),
usage=None, banner1=None, banner2=None,
display_banner=None, exit_msg=u''):
super(InteractiveShellEmbed,self).__init__(
config=config, ipython_dir=ipython_dir, user_ns=user_ns,
- user_global_ns=user_global_ns, custom_exceptions=custom_exceptions,
+ user_module=user_module, custom_exceptions=custom_exceptions,
usage=usage, banner1=banner1, banner2=banner2,
display_banner=display_banner
)
@@ -95,7 +95,7 @@ def __init__(self, config=None, ipython_dir=None, user_ns=None,
def init_sys_modules(self):
pass
- def __call__(self, header='', local_ns=None, global_ns=None, dummy=None,
+ def __call__(self, header='', local_ns=None, module=None, dummy=None,
stack_depth=1):
"""Activate the interactive interpreter.
@@ -140,14 +140,14 @@ def __call__(self, header='', local_ns=None, global_ns=None, dummy=None,
# Call the embedding code with a stack depth of 1 so it can skip over
# our call and get the original caller's namespaces.
- self.mainloop(local_ns, global_ns, stack_depth=stack_depth)
+ self.mainloop(local_ns, module, stack_depth=stack_depth)
self.banner2 = self.old_banner2
if self.exit_msg is not None:
print self.exit_msg
- def mainloop(self, local_ns=None, global_ns=None, stack_depth=0,
+ def mainloop(self, local_ns=None, module=None, stack_depth=0,
display_banner=None):
"""Embeds IPython into a running python program.
@@ -172,32 +172,37 @@ def mainloop(self, local_ns=None, global_ns=None, stack_depth=0,
there is no fundamental reason why it can't work perfectly."""
# Get locals and globals from caller
- if local_ns is None or global_ns is None:
+ if local_ns is None or module is None:
call_frame = sys._getframe(stack_depth).f_back
if local_ns is None:
local_ns = call_frame.f_locals
- if global_ns is None:
+ if module is None:
global_ns = call_frame.f_globals
-
+ module = sys.modules[global_ns['__name__']]
+
+ # Save original namespace and module so we can restore them after
+ # embedding; otherwise the shell doesn't shut down correctly.
+ orig_user_module = self.user_module
+ orig_user_ns = self.user_ns
+
# Update namespaces and fire up interpreter
-
+
# The global one is easy, we can just throw it in
- self.user_global_ns = global_ns
+ self.user_module = module
- # but the user/local one is tricky: ipython needs it to store internal
- # data, but we also need the locals. We'll copy locals in the user
- # one, but will track what got copied so we can delete them at exit.
- # This is so that a later embedded call doesn't see locals from a
- # previous call (which most likely existed in a separate scope).
- local_varnames = local_ns.keys()
- self.user_ns.update(local_ns)
- #self.user_ns['local_ns'] = local_ns # dbg
+ # But the user/local one is tricky: ipython needs it to store internal
+ # data, but we also need the locals. We'll throw our hidden variables
+ # like _ih and get_ipython() into the local namespace, but delete them
+ # later.
+ self.user_ns = local_ns
+ self.init_user_ns()
# Patch for global embedding to make sure that things don't overwrite
# user globals accidentally. Thanks to Richard <rxe@renre-europe.com>
# FIXME. Test this a bit more carefully (the if.. is new)
- if local_ns is None and global_ns is None:
+ # N.B. This can't now ever be called. Not sure what it was for.
+ if local_ns is None and module is None:
self.user_global_ns.update(__main__.__dict__)
# make sure the tab-completer has the correct frame information, so it
@@ -206,13 +211,14 @@ def mainloop(self, local_ns=None, global_ns=None, stack_depth=0,
with nested(self.builtin_trap, self.display_trap):
self.interact(display_banner=display_banner)
-
- # now, purge out the user namespace from anything we might have added
- # from the caller's local namespace
- delvar = self.user_ns.pop
- for var in local_varnames:
- delvar(var,None)
-
+
+ # now, purge out the local namespace of IPython's hidden variables.
+ for name in self.user_ns_hidden:
+ local_ns.pop(name, None)
+
+ # Restore original namespace so shell can shut down when we exit.
+ self.user_module = orig_user_module
+ self.user_ns = orig_user_ns
_embedded_shell = None
@@ -172,13 +172,13 @@ class TerminalInteractiveShell(InteractiveShell):
)
def __init__(self, config=None, ipython_dir=None, profile_dir=None, user_ns=None,
- user_global_ns=None, custom_exceptions=((),None),
+ user_module=None, custom_exceptions=((),None),
usage=None, banner1=None, banner2=None,
display_banner=None):
super(TerminalInteractiveShell, self).__init__(
config=config, profile_dir=profile_dir, user_ns=user_ns,
- user_global_ns=user_global_ns, custom_exceptions=custom_exceptions
+ user_module=user_module, custom_exceptions=custom_exceptions
)
# use os.system instead of utils.process.system by default,
# because piped system doesn't make sense in the Terminal:
@@ -190,7 +190,6 @@ def start_ipython():
# Create and initialize our test-friendly IPython instance.
shell = TerminalInteractiveShell.instance(config=config,
user_ns=ipnsdict(),
- user_global_ns={}
)
# A few more tweaks needed for playing nicely with doctests...
@@ -206,8 +205,7 @@ def start_ipython():
# can capture subcommands and print them to Python's stdout, otherwise the
# doctest machinery would miss them.
shell.system = py3compat.MethodType(xsys, shell)
-
-
+
shell._showtraceback = py3compat.MethodType(_showtraceback, shell)
# IPython is ready, now clean up some global state...
@@ -271,6 +271,8 @@ def setUp(self):
# for IPython examples *only*, we swap the globals with the ipython
# namespace, after updating it with the globals (which doctest
# fills with the necessary info from the module being tested).
+ self.user_ns_orig = {}
+ self.user_ns_orig.update(_ip.user_ns)
_ip.user_ns.update(self._dt_test.globs)
self._dt_test.globs = _ip.user_ns
# IPython must protect the _ key in the namespace (it can't exist)
@@ -286,6 +288,8 @@ def tearDown(self):
# teardown doesn't destroy the ipython namespace
if isinstance(self._dt_test.examples[0],IPExample):
self._dt_test.globs = self._dt_test_globs_ori
+ _ip.user_ns.clear()
+ _ip.user_ns.update(self.user_ns_orig)
# Restore the behavior of the '_' key in the user namespace to
# normal after each doctest, so that unittests behave normally
_ip.user_ns.protect_underscore = False
@@ -97,6 +97,10 @@ Major Bugs fixed
* IPython no longer crashes when started on recent versions of Python 3 in
Windows (:ghissue:`737`).
+* Instances of classes defined interactively can now be pickled (:ghissue:`29`;
+ :ghpull:`648`). Note that pickling saves a reference to the class definition,
+ so unpickling the instances will only work where the class has been defined.
+
.. * use bullet list
Backwards incompatible changes
@@ -132,4 +136,10 @@ Backwards incompatible changes
The full path will still work, and is necessary for using custom launchers not in
IPython's launcher module.
+* For embedding a shell, note that the parameter ``user_global_ns`` has been
+ replaced by ``user_module``, and expects a module-like object, rather than
+ a namespace dict. The ``user_ns`` parameter works the same way as before, and
+ calling :func:`~IPython.frontend.terminal.embed.embed` with no arguments still
+ works the same way.
+
.. * use bullet list

0 comments on commit a1e4911

Please sign in to comment.