From c0ad0135b4603149b2b6837a70aa87d372546c06 Mon Sep 17 00:00:00 2001
From: "Bradley M. Froehle" <brad.froehle@gmail.com>
Date: Wed, 29 Feb 2012 15:47:07 -0800
Subject: [PATCH 1/7] Update deepreload to use a rewritten knee.py. Fixes
 dreload(numpy).

knee.py, a Python re-implementation of hierarchical module import was
removed from the standard library because it no longer functioned properly.

deepreload.py is little more than a hacked version of knee.py which overrides
__builtin__.__import__ to ensure that each module is re-imported once (before
just referring to sys.modules as usual).

In addition, `os.path` was added to the default excluded modules, since
somehow it has an entry in sys.modules without `os' being a package.
---
 IPython/lib/deepreload.py | 364 ++++++++++++++++++++++++++------------
 1 file changed, 248 insertions(+), 116 deletions(-)

diff --git a/IPython/lib/deepreload.py b/IPython/lib/deepreload.py
index 81ac6bf5fc9..0f9c242f76b 100644
--- a/IPython/lib/deepreload.py
+++ b/IPython/lib/deepreload.py
@@ -14,9 +14,9 @@
 
     __builtin__.dreload = deepreload.reload
 
-This code is almost entirely based on knee.py from the standard library.
+This code is almost entirely based on knee.py, which is a Python
+re-implementation of hierarchical module import.
 """
-
 #*****************************************************************************
 #       Copyright (C) 2001 Nathaniel Gray <n8gray@caltech.edu>
 #
@@ -28,135 +28,267 @@
 import imp
 import sys
 
-# Replacement for __import__()
-def deep_import_hook(name, globals=None, locals=None, fromlist=None, level=-1):
-    # For now level is ignored, it's just there to prevent crash
-    # with from __future__ import absolute_import
-    parent = determine_parent(globals)
-    q, tail = find_head_package(parent, name)
-    m = load_tail(q, tail)
-    if not fromlist:
-        return q
-    if hasattr(m, "__path__"):
-        ensure_fromlist(m, fromlist)
-    return m
+from types import ModuleType
+from warnings import warn
+
+def get_parent(globals, level):
+    """
+    parent, name = get_parent(globals, level)
+
+    Return the package that an import is being performed in.  If globals comes
+    from the module foo.bar.bat (not itself a package), this returns the
+    sys.modules entry for foo.bar.  If globals is from a package's __init__.py,
+    the package's entry in sys.modules is returned.
+
+    If globals doesn't come from a package or a module in a package, or a
+    corresponding entry is not found in sys.modules, None is returned.
+    """
+    orig_level = level
 
-def determine_parent(globals):
-    if not globals or  not globals.has_key("__name__"):
-        return None
-    pname = globals['__name__']
-    if globals.has_key("__path__"):
-        parent = sys.modules[pname]
-        assert globals is parent.__dict__
-        return parent
-    if '.' in pname:
-        i = pname.rfind('.')
-        pname = pname[:i]
-        parent = sys.modules[pname]
-        assert parent.__name__ == pname
-        return parent
-    return None
-
-def find_head_package(parent, name):
-    # Import the first
-    if '.' in name:
-        # 'some.nested.package' -> head = 'some', tail = 'nested.package'
-        i = name.find('.')
-        head = name[:i]
-        tail = name[i+1:]
+    if not level or not isinstance(globals, dict):
+        return None, ''
+
+    pkgname = globals.get('__package__', None)
+
+    if pkgname is not None:
+        # __package__ is set, so use it
+        if not hasattr(pkgname, 'rindex'):
+            raise ValueError('__package__ set to non-string')
+        if len(pkgname) == 0:
+            if level > 0:
+                raise ValueError('Attempted relative import in non-package')
+            return None, ''
+        name = pkgname
     else:
-        # 'packagename' -> head = 'packagename', tail = ''
-        head = name
-        tail = ""
-    if parent:
-        # If this is a subpackage then qname = parent's name + head
-        qname = "%s.%s" % (parent.__name__, head)
+        # __package__ not set, so figure it out and set it
+        if '__name__' not in globals:
+            return None, ''
+        modname = globals['__name__']
+
+        if '__path__' in globals:
+            # __path__ is set, so modname is already the package name
+            globals['__package__'] = name = modname
+        else:
+            # Normal module, so work out the package name if any
+            lastdot = modname.rfind('.')
+            if lastdot < 0 and level > 0:
+                raise ValueError("Attempted relative import in non-package")
+            if lastdot < 0:
+                globals['__package__'] = None
+                return None, ''
+            globals['__package__'] = name = modname[:lastdot]
+
+    dot = len(name)
+    for x in xrange(level, 1, -1):
+        try:
+            dot = name.rindex('.', 0, dot)
+        except ValueError:
+            raise ValueError("attempted relative import beyond top-level "
+                             "package")
+    name = name[:dot]
+
+    try:
+        parent = sys.modules[name]
+    except:
+        if orig_level < 1:
+            warn("Parent module '%.200s' not found while handling absolute "
+                 "import" % name)
+            parent = None
+        else:
+            raise SystemError("Parent module '%.200s' not loaded, cannot "
+                              "perform relative import" % name)
+
+    # We expect, but can't guarantee, if parent != None, that:
+    # - parent.__name__ == name
+    # - parent.__dict__ is globals
+    # If this is violated...  Who cares?
+    return parent, name
+
+def load_next(mod, altmod, name, buf):
+    """
+    mod, name, buf = load_next(mod, altmod, name, buf)
+
+    altmod is either None or same as mod
+    """
+
+    if len(name) == 0:
+        # completely empty module name should only happen in
+        # 'from . import' (or '__import__("")')
+        return mod, None, buf
+
+    dot = name.find('.')
+    if dot == 0:
+        raise ValueError('Empty module name')
+
+    if dot < 0:
+        subname = name
+        next = None
     else:
-        qname = head
-    q = import_module(head, qname, parent)
-    if q: return q, tail
-    if parent:
-        qname = head
-        parent = None
-        q = import_module(head, qname, parent)
-        if q: return q, tail
-    raise ImportError, "No module named " + qname
-
-def load_tail(q, tail):
-    m = q
-    while tail:
-        i = tail.find('.')
-        if i < 0: i = len(tail)
-        head, tail = tail[:i], tail[i+1:]
-
-        # fperez: fix dotted.name reloading failures by changing:
-        #mname = "%s.%s" % (m.__name__, head)
-        # to:
-        mname = m.__name__
-        # This needs more testing!!! (I don't understand this module too well)
-
-        #print '** head,tail=|%s|->|%s|, mname=|%s|' % (head,tail,mname)  # dbg
-        m = import_module(head, mname, m)
-        if not m:
-            raise ImportError, "No module named " + mname
-    return m
+        subname = name[:dot]
+        next = name[dot+1:]
 
-def ensure_fromlist(m, fromlist, recursive=0):
-    for sub in fromlist:
-        if sub == "*":
-            if not recursive:
-                try:
-                    all = m.__all__
-                except AttributeError:
-                    pass
-                else:
-                    ensure_fromlist(m, all, 1)
-            continue
-        if sub != "*" and not hasattr(m, sub):
-            subname = "%s.%s" % (m.__name__, sub)
-            submod = import_module(sub, subname, m)
-            if not submod:
-                raise ImportError, "No module named " + subname
+    if buf != '':
+        buf += '.'
+    buf += subname
+
+    result = import_submodule(mod, subname, buf)
+    if result is None and mod != altmod:
+        result = import_submodule(altmod, subname, subname)
+        if result is not None:
+            buf = subname
+
+    if result is None:
+        raise ImportError("No module named %.200s" % name)
+
+    return result, next, buf
 
 # Need to keep track of what we've already reloaded to prevent cyclic evil
 found_now = {}
 
-def import_module(partname, fqname, parent):
+def import_submodule(mod, subname, fullname):
+    """m = import_submodule(mod, subname, fullname)"""
+    # Require:
+    # if mod == None: subname == fullname
+    # else: mod.__name__ + "." + subname == fullname
+
     global found_now
-    if found_now.has_key(fqname):
+    if fullname in found_now and fullname in sys.modules:
+        m = sys.modules[fullname]
+    else:
+        print 'Reloading', fullname
+        found_now[fullname] = 1
+        oldm = sys.modules.get(fullname, None)
+
+        if mod is None:
+            path = None
+        elif hasattr(mod, '__path__'):
+            path = mod.__path__
+        else:
+            return None
+
         try:
-            return sys.modules[fqname]
-        except KeyError:
-            pass
+            fp, filename, stuff  = imp.find_module(subname, path)
+        except ImportError:
+            return None
+
+        try:
+            m = imp.load_module(fullname, fp, filename, stuff)
+        except:
+            # load_module probably removed name from modules because of
+            # the error.  Put back the original module object.
+            if oldm:
+                sys.modules[fullname] = oldm
+            raise
+        finally:
+            if fp: fp.close()
+
+        add_submodule(mod, m, fullname, subname)
+
+    return m
+
+def add_submodule(mod, submod, fullname, subname):
+    """mod.{subname} = submod"""
+    if mod is None:
+        return #Nothing to do here.
+
+    if submod is None:
+        submod = sys.modules[fullname]
 
-    print 'Reloading', fqname #, sys.excepthook is sys.__excepthook__, \
-            #sys.displayhook is sys.__displayhook__
+    setattr(mod, subname, submod)
+
+    return
+
+def ensure_fromlist(mod, fromlist, buf, recursive):
+    """Handle 'from module import a, b, c' imports."""
+    if not hasattr(mod, '__path__'):
+        return
+    for item in fromlist:
+        if not hasattr(item, 'rindex'):
+            raise TypeError("Item in ``from list'' not a string")
+        if item == '*':
+            if recursive:
+                continue # avoid endless recursion
+            try:
+                all = mod.__all__
+            except AttributeError:
+                pass
+            else:
+                ret = ensure_fromlist(mod, all, buf, 1)
+                if not ret:
+                    return 0
+        elif not hasattr(mod, item):
+            import_submodule(mod, item, buf + '.' + item)
+
+def import_module_level(name, globals=None, locals=None, fromlist=None, level=-1):
+    """Replacement for __import__()"""
+    parent, buf = get_parent(globals, level)
+
+    head, name, buf = load_next(parent, None if level < 0 else parent, name, buf)
+
+    tail = head
+    while name:
+        tail, name, buf = load_next(tail, tail, name, buf)
+
+    # If tail is None, both get_parent and load_next found
+    # an empty module name: someone called __import__("") or
+    # doctored faulty bytecode
+    if tail is None:
+        raise ValueError('Empty module name')
+
+    if not fromlist:
+        return head
 
-    found_now[fqname] = 1
+    ensure_fromlist(tail, fromlist, buf, 0)
+    return tail
+
+modules_reloading = {}
+
+def reload_module(m):
+    """Replacement for reload()."""
+    if not isinstance(m, ModuleType):
+        raise TypeError("reload() argument must be module")
+
+    name = m.__name__
+
+    if name not in sys.modules:
+        raise ImportError("reload(): module %.200s not in sys.modules" % name)
+
+    global modules_reloading
     try:
-        fp, pathname, stuff = imp.find_module(partname,
-                                              parent and parent.__path__)
-    except ImportError:
-        return None
+        return modules_reloading[name]
+    except:
+        modules_reloading[name] = m
+
+    dot = name.rfind('.')
+    if dot < 0:
+        subname = name
+        path = None
+    else:
+        try:
+            parent = sys.modules[name[:dot]]
+        except KeyError:
+            modules_reloading.clear()
+            raise ImportError("reload(): parent %.200s not in sys.modules" % name[:dot])
+        subname = name[dot+1:]
+        path = getattr(parent, "__path__", None)
 
     try:
-        m = imp.load_module(fqname, fp, pathname, stuff)
+        fp, filename, stuff  = imp.find_module(subname, path)
     finally:
-        if fp: fp.close()
+        modules_reloading.clear()
 
-    if parent:
-        setattr(parent, partname, m)
-
-    return m
+    try:
+        newm = imp.load_module(name, fp, filename, stuff)
+    except:
+         # load_module probably removed name from modules because of
+         # the error.  Put back the original module object.
+        sys.modules[name] = m
+        raise
+    finally:
+        if fp: fp.close()
 
-def deep_reload_hook(module):
-    name = module.__name__
-    if '.' not in name:
-        return import_module(name, name, None)
-    i = name.rfind('.')
-    pname = name[:i]
-    parent = sys.modules[pname]
-    return import_module(name[i+1:], name, parent)
+    modules_reloading.clear()
+    return newm
 
 # Save the original hooks
 try:
@@ -165,7 +297,7 @@ def deep_reload_hook(module):
     original_reload = imp.reload    # Python 3
 
 # Replacement for reload()
-def reload(module, exclude=['sys', '__builtin__', '__main__']):
+def reload(module, exclude=['sys', 'os.path', '__builtin__', '__main__']):
     """Recursively reload all modules used in the given module.  Optionally
     takes a list of modules to exclude from reloading.  The default exclude
     list contains sys, __main__, and __builtin__, to prevent, e.g., resetting
@@ -175,9 +307,9 @@ def reload(module, exclude=['sys', '__builtin__', '__main__']):
     for i in exclude:
         found_now[i] = 1
     original_import = __builtin__.__import__
-    __builtin__.__import__ = deep_import_hook
+    __builtin__.__import__ = import_module_level
     try:
-        ret = deep_reload_hook(module)
+        ret = reload_module(module)
     finally:
         __builtin__.__import__ = original_import
         found_now = {}

From 47c768ca56442b66f05237d470c1af6d691e81d2 Mon Sep 17 00:00:00 2001
From: "Bradley M. Froehle" <brad.froehle@gmail.com>
Date: Tue, 17 Apr 2012 09:56:41 -0700
Subject: [PATCH 2/7] Keep original function names.

---
 IPython/lib/deepreload.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/IPython/lib/deepreload.py b/IPython/lib/deepreload.py
index 0f9c242f76b..b9286faed1b 100644
--- a/IPython/lib/deepreload.py
+++ b/IPython/lib/deepreload.py
@@ -219,7 +219,7 @@ def ensure_fromlist(mod, fromlist, buf, recursive):
         elif not hasattr(mod, item):
             import_submodule(mod, item, buf + '.' + item)
 
-def import_module_level(name, globals=None, locals=None, fromlist=None, level=-1):
+def deep_import_hook(name, globals=None, locals=None, fromlist=None, level=-1):
     """Replacement for __import__()"""
     parent, buf = get_parent(globals, level)
 
@@ -243,7 +243,7 @@ def import_module_level(name, globals=None, locals=None, fromlist=None, level=-1
 
 modules_reloading = {}
 
-def reload_module(m):
+def deep_reload_hook(m):
     """Replacement for reload()."""
     if not isinstance(m, ModuleType):
         raise TypeError("reload() argument must be module")
@@ -307,9 +307,9 @@ def reload(module, exclude=['sys', 'os.path', '__builtin__', '__main__']):
     for i in exclude:
         found_now[i] = 1
     original_import = __builtin__.__import__
-    __builtin__.__import__ = import_module_level
+    __builtin__.__import__ = deep_import_hook
     try:
-        ret = reload_module(module)
+        ret = deep_reload_hook(module)
     finally:
         __builtin__.__import__ = original_import
         found_now = {}

From b0f53e41bdf7b84d16ea68e4bf52c184ec7e5009 Mon Sep 17 00:00:00 2001
From: "Bradley M. Froehle" <brad.froehle@gmail.com>
Date: Tue, 17 Apr 2012 10:12:46 -0700
Subject: [PATCH 3/7] Add deepreload unit test.

Thanks to Fernando for the suggestion and starting code.
---
 IPython/lib/tests/test_deepreload.py | 15 +++++++++++++++
 1 file changed, 15 insertions(+)
 create mode 100644 IPython/lib/tests/test_deepreload.py

diff --git a/IPython/lib/tests/test_deepreload.py b/IPython/lib/tests/test_deepreload.py
new file mode 100644
index 00000000000..af2e4440a86
--- /dev/null
+++ b/IPython/lib/tests/test_deepreload.py
@@ -0,0 +1,15 @@
+"""Test suite for the deepreload module."""
+
+from IPython.testing import decorators as dec
+from IPython.lib.deepreload import reload as dreload
+
+@dec.skipif_not_numpy
+def test_deepreload_numpy():
+    import numpy
+    exclude = [
+        # Standard exclusions:
+        'sys', 'os.path', '__builtin__', '__main__',
+        # Test-related exclusions:
+        'unittest',
+        ]
+    dreload(numpy, exclude=exclude)

From 27ee2752a71ee415154c40e1978edb9d5221a331 Mon Sep 17 00:00:00 2001
From: "Bradley M. Froehle" <brad.froehle@gmail.com>
Date: Tue, 17 Apr 2012 11:49:59 -0700
Subject: [PATCH 4/7] Reformat test to a standard style.

---
 IPython/lib/tests/test_deepreload.py | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/IPython/lib/tests/test_deepreload.py b/IPython/lib/tests/test_deepreload.py
index af2e4440a86..c994c988521 100644
--- a/IPython/lib/tests/test_deepreload.py
+++ b/IPython/lib/tests/test_deepreload.py
@@ -1,8 +1,17 @@
+# -*- coding: utf-8 -*-
 """Test suite for the deepreload module."""
 
+#-----------------------------------------------------------------------------
+# Imports
+#-----------------------------------------------------------------------------
+
 from IPython.testing import decorators as dec
 from IPython.lib.deepreload import reload as dreload
 
+#-----------------------------------------------------------------------------
+# Test functions begin
+#-----------------------------------------------------------------------------
+
 @dec.skipif_not_numpy
 def test_deepreload_numpy():
     import numpy

From 45df6ff65a5b0e71e969487a0ca86d715736e887 Mon Sep 17 00:00:00 2001
From: "Bradley M. Froehle" <brad.froehle@gmail.com>
Date: Tue, 17 Apr 2012 12:12:04 -0700
Subject: [PATCH 5/7] Add deepreload functionality test.

---
 IPython/lib/tests/test_deepreload.py | 28 ++++++++++++++++++++++++++++
 1 file changed, 28 insertions(+)

diff --git a/IPython/lib/tests/test_deepreload.py b/IPython/lib/tests/test_deepreload.py
index c994c988521..ee55f5bb762 100644
--- a/IPython/lib/tests/test_deepreload.py
+++ b/IPython/lib/tests/test_deepreload.py
@@ -5,7 +5,13 @@
 # Imports
 #-----------------------------------------------------------------------------
 
+import os
+import sys
+
+import nose.tools as nt
+
 from IPython.testing import decorators as dec
+from IPython.utils.tempdir import TemporaryDirectory
 from IPython.lib.deepreload import reload as dreload
 
 #-----------------------------------------------------------------------------
@@ -14,6 +20,7 @@
 
 @dec.skipif_not_numpy
 def test_deepreload_numpy():
+    "Test that NumPy can be deep reloaded."
     import numpy
     exclude = [
         # Standard exclusions:
@@ -22,3 +29,24 @@ def test_deepreload_numpy():
         'unittest',
         ]
     dreload(numpy, exclude=exclude)
+
+def test_deepreload():
+    "Test that dreload does deep reloads and skips excluded modules."
+    with TemporaryDirectory() as tmpdir:
+        sys.path.insert(0, tmpdir)
+        with open(os.path.join(tmpdir, 'A.py'), 'w') as f:
+            f.write("class Object(object):\n    pass\n")
+        with open(os.path.join(tmpdir, 'B.py'), 'w') as f:
+            f.write("import A\n")
+        import A
+        import B
+
+        # Test that A is not reloaded.
+        obj = A.Object()
+        dreload(B, exclude=['A'])
+        nt.assert_is_instance(obj, A.Object)
+
+        # Test that A is reloaded.
+        obj = A.Object()
+        dreload(B)
+        nt.assert_not_is_instance(obj, A.Object)

From 807f6092740a5b0e762fa70c2a39677762d54b0b Mon Sep 17 00:00:00 2001
From: "Bradley M. Froehle" <brad.froehle@gmail.com>
Date: Tue, 17 Apr 2012 17:25:02 -0700
Subject: [PATCH 6/7] Fix deepreload tests in Python 2.6.

Apparently assert_is_instance and assert_not_is_instance are not available
in Python 2.6.
---
 IPython/lib/tests/test_deepreload.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/IPython/lib/tests/test_deepreload.py b/IPython/lib/tests/test_deepreload.py
index ee55f5bb762..b08b55ab867 100644
--- a/IPython/lib/tests/test_deepreload.py
+++ b/IPython/lib/tests/test_deepreload.py
@@ -44,9 +44,9 @@ def test_deepreload():
         # Test that A is not reloaded.
         obj = A.Object()
         dreload(B, exclude=['A'])
-        nt.assert_is_instance(obj, A.Object)
+        nt.assert_true(isinstance(obj, A.Object))
 
         # Test that A is reloaded.
         obj = A.Object()
         dreload(B)
-        nt.assert_not_is_instance(obj, A.Object)
+        nt.assert_false(isinstance(obj, A.Object))

From 2e6a444ff1b951c2180c70b32658ba73c78622b7 Mon Sep 17 00:00:00 2001
From: "Bradley M. Froehle" <brad.froehle@gmail.com>
Date: Tue, 17 Apr 2012 17:26:08 -0700
Subject: [PATCH 7/7] Clean up sys.path entry.

---
 IPython/lib/tests/test_deepreload.py | 35 ++++++++++++++--------------
 1 file changed, 18 insertions(+), 17 deletions(-)

diff --git a/IPython/lib/tests/test_deepreload.py b/IPython/lib/tests/test_deepreload.py
index b08b55ab867..e8d8e05481b 100644
--- a/IPython/lib/tests/test_deepreload.py
+++ b/IPython/lib/tests/test_deepreload.py
@@ -11,6 +11,7 @@
 import nose.tools as nt
 
 from IPython.testing import decorators as dec
+from IPython.utils.syspathcontext import prepended_to_syspath
 from IPython.utils.tempdir import TemporaryDirectory
 from IPython.lib.deepreload import reload as dreload
 
@@ -33,20 +34,20 @@ def test_deepreload_numpy():
 def test_deepreload():
     "Test that dreload does deep reloads and skips excluded modules."
     with TemporaryDirectory() as tmpdir:
-        sys.path.insert(0, tmpdir)
-        with open(os.path.join(tmpdir, 'A.py'), 'w') as f:
-            f.write("class Object(object):\n    pass\n")
-        with open(os.path.join(tmpdir, 'B.py'), 'w') as f:
-            f.write("import A\n")
-        import A
-        import B
-
-        # Test that A is not reloaded.
-        obj = A.Object()
-        dreload(B, exclude=['A'])
-        nt.assert_true(isinstance(obj, A.Object))
-
-        # Test that A is reloaded.
-        obj = A.Object()
-        dreload(B)
-        nt.assert_false(isinstance(obj, A.Object))
+        with prepended_to_syspath(tmpdir):
+            with open(os.path.join(tmpdir, 'A.py'), 'w') as f:
+                f.write("class Object(object):\n    pass\n")
+            with open(os.path.join(tmpdir, 'B.py'), 'w') as f:
+                f.write("import A\n")
+            import A
+            import B
+
+            # Test that A is not reloaded.
+            obj = A.Object()
+            dreload(B, exclude=['A'])
+            nt.assert_true(isinstance(obj, A.Object))
+
+            # Test that A is reloaded.
+            obj = A.Object()
+            dreload(B)
+            nt.assert_false(isinstance(obj, A.Object))