Initial commit

README.md

@@ -1,2 +1,32 @@
-# modulepackage
-Materials for PyCon2015 Tutorial "Modules and Packages : Live and Let Die"
+# Modules and Packages : Live and Let Die
+
+*Tutorial Presentation at PyCon'2015.  April 9, 2015.  Montreal.*
+
+This tutorial assumes the use of Python 3.4 or newer.  Certain examples
+in sections 8 and 9 require the use of Python 3.5. 
+
+Course Notes are available in the file `ModulePackage.pdf`
+
+## Part 1 - Basic Knowledge
+
+## Part 2 - Packages
+
+`package_assembly/` : An example of assembling a package from submodules
+by exporting symbols in `__init__.py` files.
+
+`decorator_assembly/`: Assemble a package from submodules using a special
+`@export` decortor.
+
+## Part 3 - __main__
+
+## Part 4 - sys.path
+
+## Part 5 - Namespace Packages
+
+## Part 6 - The Module
+
+## Part 7 - The Module Reloaded
+
+## Part 8 - Import Hooks
+
+## Part 9 - Path Hooks

autoinstall/autoinstall.py

@@ -0,0 +1,22 @@
+# Import hook that autoinstall packages
+# To test this, suggest using a virtual env
+
+import sys
+import subprocess
+import importlib.util
+
+class AutoInstall(object):
+    _loaded = set()
+    @classmethod
+    def find_spec(cls, name, path, target=None):
+        if path is None and name not in cls._loaded:
+            cls._loaded.add(name)
+            print("Installing",name)
+            try:
+                out = subprocess.check_output([sys.executable, '-m', 'pip', 'install', name])
+                return importlib.util.find_spec(name)
+            except Exception as e:
+                print("Failed")
+        return None
+
+sys.meta_path.append(AutoInstall)

check_import/foo.py

@@ -0,0 +1,4 @@
+# foo.py
+import bar
+
+print('imported foo')

check_import/simplefoo.py

@@ -0,0 +1,3 @@
+# simplefoo.py
+
+print('imported simplefoo')

check_import/spam.py

@@ -0,0 +1,9 @@
+# spam.py
+
+from importlib.util import find_spec
+
+if find_spec('foo'):
+    import foo
+else:
+    import simplefoo as foo
+

decorator_assembly/spam/__init__.py

@@ -0,0 +1,11 @@
+# spam/__init__.py
+
+__all__ = []
+
+def export(defn):
+    globals()[defn.__name__] = defn
+    __all__.append(defn.__name__)
+    return defn
+
+from .foo import *
+from .bar import *

decorator_assembly/spam/bar.py

@@ -0,0 +1,9 @@
+# bar.py
+
+from . import export
+
+@export
+class Bar(object):
+    pass
+
+print('bar imported')

decorator_assembly/spam/foo.py

@@ -0,0 +1,9 @@
+# foo.py
+
+from . import export
+
+@export
+class Foo(object):
+    pass
+
+print('foo imported')

import_patch/import_patch.py

@@ -0,0 +1,7 @@
+import builtins
+
+def my_import(modname, *args, imp=__import__):
+    print('importing', modname)
+    return imp(modname, *args)
+
+builtins.__import__ = my_import

lazy_assembly/spam/__init__.py

@@ -0,0 +1,32 @@
+# spam/__init__.py
+
+# List the exported symbols by module
+_submodule_exports = {
+    '.foo' : ['Foo'],
+    '.bar' : ['Bar']
+}
+
+# Make a {name: modname } mapping
+_submodule_by_name = {
+     name: modulename 
+           for modulename in _submodule_exports
+           for name in _submodule_exports[modulename] }
+
+import types, sys, importlib
+
+class OnDemandModule(types.ModuleType):
+    def __getattr__(self, name):
+        modulename = _submodule_by_name.get(name)
+        if modulename:
+            module = importlib.import_module(modulename, 
+                                             __package__)
+            print('Loaded', name)
+            value = getattr(module, name)
+            setattr(self, name, value)
+            return value
+        raise AttributeError('No attribute %s' % name)
+
+newmodule = OnDemandModule(__name__)
+newmodule.__dict__.update(globals())
+newmodule.__all__ = list(_submodule_by_name)
+sys.modules[__name__] = newmodule

lazy_assembly/spam/bar.py

@@ -0,0 +1,8 @@
+# bar.py
+
+__all__ = ['Bar']
+
+class Bar(object):
+    pass
+
+print('bar imported')

lazy_assembly/spam/foo.py

@@ -0,0 +1,8 @@
+# foo.py
+
+__all__ = ['Foo']
+
+class Foo(object):
+    pass
+
+print('foo imported')

lazy_import/lazy_import.py

@@ -0,0 +1,38 @@
+import types
+import importlib.util
+import sys
+
+class _Module(types.ModuleType):
+    pass
+
+class _LazyModule(_Module):
+    def __init__(self, spec):
+        super().__init__(spec.name)
+        self.__file__ =spec.origin
+        self.__package__ = spec.parent
+        self.__loader__= spec.loader
+        self.__path__ = spec.submodule_search_locations
+        self.__spec__ =spec
+
+    def __getattr__(self, name):
+        self.__class__ = _Module
+        self.__spec__.loader.exec_module(self)
+        assert sys.modules[self.__name__] == self
+        return getattr(self, name)
+
+def lazy_import(name):
+    # If already loaded, return the module
+    if name in sys.modules:
+        return sys.modules[name]
+
+    # Not loaded. Find the spec
+    spec = importlib.util.find_spec(name)
+    if not spec:
+       raise ImportError('No module %r' % name)
+
+    # Check for compatibility
+    if not hasattr(spec.loader, 'exec_module'):
+       raise ImportError('Not supported')
+
+    module = sys.modules[name] = _LazyModule(spec)
+    return module    

main_wrapper/script.py

@@ -0,0 +1,3 @@
+# script.py
+
+print("I'm a script")

main_wrapper/tool.py

@@ -0,0 +1,25 @@
+# tool.py
+
+print("I'm a tool.")
+
+import sys
+import os.path
+
+def main():
+    if len(sys.argv) < 2:
+        raise SystemExit('Usage: python3 -m tool script.py')
+    sys.argv[:] = sys.argv[1:]
+    progname = sys.argv[0]
+    sys.path.insert(0, os.path.dirname(progname))
+    with open(progname, 'rb') as fp:
+         code = compile(fp.read(), progname, 'exec')
+    globs = {
+         '__file__' : progname,
+         '__name__' : '__main__',
+         '__package__' : None,
+         '__cached__' : None
+    }
+    exec(code, globs)
+
+if __name__ == '__main__':
+    main()

mini_import/mini_imp.py

@@ -0,0 +1,25 @@
+# mini_imp.py
+#
+# A tiny implementation of "import"
+
+import types
+import sys
+
+def import_module(modname):
+    # Check the module cache
+    if modname in sys.modules:
+        return sys.modules[modname]
+
+    sourcepath = modname + '.py'
+    with open(sourcepath, 'r') as f:
+         sourcecode = f.read()
+    mod = types.ModuleType(modname)
+    mod.__file__ = sourcepath
+    code = compile(sourcecode, sourcepath, 'exec')
+
+    # Insert into the module cache prior to exec
+    sys.modules[modname] = mod
+    exec(code, mod.__dict__)
+
+    # Return from the module cache
+    return sys.modules[modname]

mini_import/runme.py

@@ -0,0 +1,5 @@
+from mini_imp import import_module
+
+spam = import_module('spam')
+spam.foo()
+spam.bar()

mini_import/spam.py

@@ -0,0 +1,10 @@
+# spam.py
+
+print('imported spam')
+
+def foo():
+    print('spam.foo')
+
+def bar():
+    print('spam.bar')
+

namespace_package/runme.py

@@ -0,0 +1,10 @@
+# Illustrate how two packages become one if there's no __init__
+
+import sys
+sys.path.extend(['spam_foo', 'spam_bar'])
+
+import spam.foo
+import spam.bar
+
+print(spam.foo.__file__)
+print(spam.bar.__file__)

namespace_package/spam_bar/spam/bar.py

@@ -0,0 +1,6 @@
+# bar.py
+
+class Bar(object):
+    pass
+
+print('imported bar')

namespace_package/spam_foo/spam/foo.py

@@ -0,0 +1,6 @@
+# foo.py
+
+class Foo(object):
+    pass
+
+print('imported foo')

package_assembly/spam/__init__.py

@@ -0,0 +1,6 @@
+# spam/__init__.py
+
+from .foo import *
+from .bar import *
+
+__all__ = (foo.__all__ + bar.__all__)

package_assembly/spam/bar.py

@@ -0,0 +1,8 @@
+# bar.py
+
+__all__ = ['Bar']
+
+class Bar(object):
+    pass
+
+print('bar imported')

package_assembly/spam/foo.py

@@ -0,0 +1,8 @@
+# foo.py
+
+__all__ = ['Foo']
+
+class Foo(object):
+    pass
+
+print('foo imported')

redisimport/redisloader.py

@@ -0,0 +1,31 @@
+# redisloader.py                                                                                                                                   
+import redis
+import importlib.util
+
+class RedisImporter(object):
+    def __init__(self, *args, **kwargs):
+        self.conn = redis.Redis(*args, **kwargs)
+        self.conn.exists('test')
+
+    def find_spec(self, name, path, target=None):
+        origin = name + '.py'
+        if self.conn.exists(origin):
+            loader = RedisLoader(origin, self.conn)
+            return importlib.util.spec_from_loader(name, loader)
+        return None
+
+def enable(*args, **kwargs):
+    import sys
+    sys.meta_path.insert(0, RedisImporter(*args, **kwargs))
+
+class RedisLoader(object):
+    def __init__(self, origin, conn):
+        self.origin = origin
+        self.conn = conn
+
+    def create_module(self, spec):
+        return None
+
+    def exec_module(self, module):
+        code = self.conn.get(self.origin)
+        exec(code, module.__dict__)

redisimport/runme.py

@@ -0,0 +1,12 @@
+# Test the redisloader
+
+import redisloader
+import redis
+
+redisloader.enable()
+
+r = redis.Redis()
+r.set('foo.py', 'print("imported foo")')
+
+import foo
+print(foo)

subpackage_cycle/spam/bar.py

@@ -0,0 +1,9 @@
+# bar.py
+
+try:
+    from . import foo
+except ImportError:
+    import sys
+    foo = sys.modules[__package__ + '.foo']
+
+print('imported spam.bar')

subpackage_cycle/spam/foo.py

@@ -0,0 +1,10 @@
+# foo.py
+
+try:
+    from . import bar
+except ImportError:
+    import sys
+    bar = sys.modules[__package__ + '.bar']
+
+print('imported spam.foo')
+