From 20fc5fa8bc282cc977f1c170436f38bd8094464d Mon Sep 17 00:00:00 2001 From: Erik Welch Date: Sun, 12 Jun 2016 10:42:21 -0500 Subject: [PATCH 1/8] Experimenting with `tlz` pseudo-package that copies `cytoolz` if available or `toolz`. Currently, in order to do `import tlz`, `toolz` needs to have already been loaded. --- .travis.yml | 1 - toolz/__init__.py | 3 ++ toolz/_tlz.py | 73 +++++++++++++++++++++++++++++++++++++++++ toolz/tests/test_tlz.py | 49 +++++++++++++++++++++++++++ 4 files changed, 125 insertions(+), 1 deletion(-) create mode 100644 toolz/_tlz.py create mode 100644 toolz/tests/test_tlz.py diff --git a/.travis.yml b/.travis.yml index 2a24ac3a..f389d32d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,7 +1,6 @@ sudo: false language: python python: - - "2.6" - "2.7" - "3.3" - "3.4" diff --git a/toolz/__init__.py b/toolz/__init__.py index 43226df7..12bc628f 100644 --- a/toolz/__init__.py +++ b/toolz/__init__.py @@ -19,4 +19,7 @@ functoolz._sigs.create_signature_registry() +from . import _tlz +del _tlz + __version__ = '0.8.0' diff --git a/toolz/_tlz.py b/toolz/_tlz.py new file mode 100644 index 00000000..3bcf9b8b --- /dev/null +++ b/toolz/_tlz.py @@ -0,0 +1,73 @@ +import sys +import types +from importlib import import_module + + +class TlzLoader(object): + """ Finds and loads modules when added to sys.meta_path""" + def _load_toolz(self, fullname): + package, dot, submodules = fullname.partition('.') + module_name = ''.join(['cytoolz', dot, submodules]) + try: + module = import_module(module_name) + except ImportError: + module_name = ''.join(['toolz', dot, submodules]) + module = import_module(module_name) + return module + + def find_module(self, fullname, path=None): # pragma: py3 no cover + package, dot, submodules = fullname.partition('.') + if package == 'tlz': + return self + + def load_module(self, fullname): # pragma: py3 no cover + if fullname in sys.modules: # pragma: no cover + return sys.modules[fullname] + spec = TlzSpec(fullname, self) + module = self.create_module(spec) + sys.modules[fullname] = module + self.exec_module(module) + return module + + def find_spec(self, fullname, path, target=None): # pragma: py2 no cover + package, dot, submodules = fullname.partition('.') + if package == 'tlz': + return TlzSpec(fullname, self) + + def create_module(self, spec): + module = types.ModuleType(spec.name) + return module + + def exec_module(self, module): + toolz_module = self._load_toolz(module.__name__) + d = dict(toolz_module.__dict__, **module.__dict__) + module.__dict__.update(d) + package = toolz_module.__dict__['__package__'] + if package is not None: + package, dot, submodules = package.partition('.') + module.__dict__['__package__'] = ''.join(['tlz', dot, submodules]) + + for k, v in toolz_module.__dict__.items(): + if ( + isinstance(v, types.ModuleType) + and v.__package__ == toolz_module.__name__ + ): + package, dot, submodules = v.__name__.partition('.') + module_name = ''.join(['tlz', dot, submodules]) + submodule = import_module(module_name) + module.__dict__[k] = submodule + + +class TlzSpec(object): + def __init__(self, name, loader): + self.name = name + self.loader = loader + self.origin = None + self.submodule_search_locations = [] + self.loader_state = None + self.cached = None + self.parent = None + self.has_location = False + + +sys.meta_path.append(TlzLoader()) diff --git a/toolz/tests/test_tlz.py b/toolz/tests/test_tlz.py new file mode 100644 index 00000000..ed691204 --- /dev/null +++ b/toolz/tests/test_tlz.py @@ -0,0 +1,49 @@ +import toolz + + +def test_tlz(): + import tlz + tlz.curry + tlz.functoolz.curry + assert tlz.__package__ == 'tlz' + assert tlz.__name__ == 'tlz' + import tlz.curried + print(tlz.curried.__package__) + assert tlz.curried.__package__ == 'tlz.curried' + assert tlz.curried.__name__ == 'tlz.curried' + tlz.curried.curry + import tlz.curried.operator + assert tlz.curried.operator.__package__ in (None, 'tlz.curried') + assert tlz.curried.operator.__name__ == 'tlz.curried.operator' + assert tlz.functoolz.__name__ == 'tlz.functoolz' + m1 = tlz.functoolz + import tlz.functoolz as m2 + assert m1 is m2 + import tlz.sandbox + try: + import tlzthisisabadname.curried + 1/0 + except ImportError: + pass + try: + import tlz.curry + 1/0 + except ImportError: + pass + try: + import tlz.badsubmodulename + 1/0 + except ImportError: + pass + + assert toolz.__package__ == 'toolz' + assert toolz.curried.__package__ == 'toolz.curried' + assert toolz.functoolz.__name__ == 'toolz.functoolz' + try: + import cytoolz + assert cytoolz.__package__ == 'cytoolz' + #import cytoolz.curried + assert cytoolz.curried.__package__ == 'cytoolz.curried' + assert cytoolz.functoolz.__name__ == 'cytoolz.functoolz' + except ImportError: + pass From e66fbe1b22aa8f85e319534937e97cdca9580900 Mon Sep 17 00:00:00 2001 From: Erik Welch Date: Sun, 12 Jun 2016 11:43:45 -0500 Subject: [PATCH 2/8] Update coverage for Python 3.3 --- toolz/_tlz.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/toolz/_tlz.py b/toolz/_tlz.py index 3bcf9b8b..816dc1b9 100644 --- a/toolz/_tlz.py +++ b/toolz/_tlz.py @@ -29,7 +29,7 @@ def load_module(self, fullname): # pragma: py3 no cover self.exec_module(module) return module - def find_spec(self, fullname, path, target=None): # pragma: py2 no cover + def find_spec(self, fullname, path, target=None): # pragma: no cover package, dot, submodules = fullname.partition('.') if package == 'tlz': return TlzSpec(fullname, self) From 89a4ce165d3e8f133646d744901dbaaa5e6a7fb4 Mon Sep 17 00:00:00 2001 From: Erik Welch Date: Sun, 12 Jun 2016 19:04:24 -0500 Subject: [PATCH 3/8] Always get some items from `toolz` and always introspect files from `toolz`. Hence, if you do `tlz.functoolz??` in IPython, you'll see `toolz.functoolz`. --- toolz/_tlz.py | 48 ++++++++++++++++++++++++++++++----------- toolz/tests/test_tlz.py | 5 +++++ 2 files changed, 41 insertions(+), 12 deletions(-) diff --git a/toolz/_tlz.py b/toolz/_tlz.py index 816dc1b9..cb3d7112 100644 --- a/toolz/_tlz.py +++ b/toolz/_tlz.py @@ -1,19 +1,32 @@ import sys import types from importlib import import_module +import toolz class TlzLoader(object): """ Finds and loads modules when added to sys.meta_path""" + def __init__(self): + self.always_from_toolz = set([ + toolz.pipe, + ]) + def _load_toolz(self, fullname): + rv = {} package, dot, submodules = fullname.partition('.') - module_name = ''.join(['cytoolz', dot, submodules]) try: - module = import_module(module_name) + module_name = ''.join(['cytoolz', dot, submodules]) + rv['cytoolz'] = import_module(module_name) except ImportError: + pass + try: module_name = ''.join(['toolz', dot, submodules]) - module = import_module(module_name) - return module + rv['toolz'] = import_module(module_name) + except ImportError: + pass + if not rv: + raise ImportError(fullname) + return rv def find_module(self, fullname, path=None): # pragma: py3 no cover package, dot, submodules = fullname.partition('.') @@ -39,18 +52,29 @@ def create_module(self, spec): return module def exec_module(self, module): - toolz_module = self._load_toolz(module.__name__) - d = dict(toolz_module.__dict__, **module.__dict__) - module.__dict__.update(d) - package = toolz_module.__dict__['__package__'] + toolz_mods = self._load_toolz(module.__name__) + fast_mod = toolz_mods.get('cytoolz') or toolz_mods['toolz'] + slow_mod = toolz_mods.get('toolz') or toolz_mods['cytoolz'] + module.__dict__.update(toolz.merge(fast_mod.__dict__, module.__dict__)) + package = fast_mod.__package__ if package is not None: package, dot, submodules = package.partition('.') - module.__dict__['__package__'] = ''.join(['tlz', dot, submodules]) + module.__package__ = ''.join(['tlz', dot, submodules]) + + # show file from toolz during introspection + module.__file__ = slow_mod.__file__ - for k, v in toolz_module.__dict__.items(): - if ( + for k, v in fast_mod.__dict__.items(): + tv = slow_mod.__dict__.get(k) + try: + hash(tv) + except TypeError: + tv = None + if tv in self.always_from_toolz: + module.__dict__[k] = tv + elif ( isinstance(v, types.ModuleType) - and v.__package__ == toolz_module.__name__ + and v.__package__ == fast_mod.__name__ ): package, dot, submodules = v.__name__.partition('.') module_name = ''.join(['tlz', dot, submodules]) diff --git a/toolz/tests/test_tlz.py b/toolz/tests/test_tlz.py index ed691204..2eebc0b1 100644 --- a/toolz/tests/test_tlz.py +++ b/toolz/tests/test_tlz.py @@ -47,3 +47,8 @@ def test_tlz(): assert cytoolz.functoolz.__name__ == 'cytoolz.functoolz' except ImportError: pass + + assert tlz.__file__ == toolz.__file__ + assert tlz.functoolz.__file__ == toolz.functoolz.__file__ + + assert tlz.pipe is toolz.pipe From 4965ad333d40afea1b586d4dd63b754a2dd82889 Mon Sep 17 00:00:00 2001 From: Erik Welch Date: Sun, 12 Jun 2016 19:33:49 -0500 Subject: [PATCH 4/8] Move `tlz` into a separate package that gets installed by setup.py. Test Python 2.6 again, but exclude `tlz`. --- .travis.yml | 3 ++- setup.py | 3 ++- tlz/__init__.py | 1 + toolz/_tlz.py => tlz/_build_tlz.py | 9 +++++++-- toolz/__init__.py | 3 --- toolz/tests/test_tlz.py | 9 +++++++++ 6 files changed, 21 insertions(+), 7 deletions(-) create mode 100644 tlz/__init__.py rename toolz/_tlz.py => tlz/_build_tlz.py (93%) diff --git a/.travis.yml b/.travis.yml index f389d32d..883d5fe0 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,6 +1,7 @@ sudo: false language: python python: + - "2.6" - "2.7" - "3.3" - "3.4" @@ -20,7 +21,7 @@ install: script: - export MAJOR_PYTHON_VERSION=`echo $TRAVIS_PYTHON_VERSION | cut -c 1` - coverage run --source=toolz $(which nosetests) - --with-doctest + --with-doctest toolz/ - if [[ $TRAVIS_PYTHON_VERSION != pypy* ]]; then coverage report --show-missing --fail-under=100 ; fi - if [[ $TRAVIS_PYTHON_VERSION != pypy* ]]; then pep8 --ignore=$PEP8_IGNORE --exclude=conf.py,tests,examples,bench -r --show-source . ; fi diff --git a/setup.py b/setup.py index a45d84ca..3c0b0882 100755 --- a/setup.py +++ b/setup.py @@ -15,7 +15,8 @@ keywords='functional utility itertools functools', packages=['toolz', 'toolz.sandbox', - 'toolz.curried'], + 'toolz.curried', + 'tlz'], package_data={'toolz': ['tests/*.py']}, long_description=(open('README.rst').read() if exists('README.rst') else ''), diff --git a/tlz/__init__.py b/tlz/__init__.py new file mode 100644 index 00000000..12707d9b --- /dev/null +++ b/tlz/__init__.py @@ -0,0 +1 @@ +from . import _build_tlz diff --git a/toolz/_tlz.py b/tlz/_build_tlz.py similarity index 93% rename from toolz/_tlz.py rename to tlz/_build_tlz.py index cb3d7112..52969b41 100644 --- a/toolz/_tlz.py +++ b/tlz/_build_tlz.py @@ -1,6 +1,9 @@ import sys import types -from importlib import import_module +try: + from importlib import import_module +except ImportError: + raise ImportError('"tlz" package is not available in Python 2.6') import toolz @@ -94,4 +97,6 @@ def __init__(self, name, loader): self.has_location = False -sys.meta_path.append(TlzLoader()) +tlz_loader = TlzLoader() +sys.meta_path.append(tlz_loader) +tlz_loader.exec_module(sys.modules['tlz']) diff --git a/toolz/__init__.py b/toolz/__init__.py index 12bc628f..43226df7 100644 --- a/toolz/__init__.py +++ b/toolz/__init__.py @@ -19,7 +19,4 @@ functoolz._sigs.create_signature_registry() -from . import _tlz -del _tlz - __version__ = '0.8.0' diff --git a/toolz/tests/test_tlz.py b/toolz/tests/test_tlz.py index 2eebc0b1..865562e9 100644 --- a/toolz/tests/test_tlz.py +++ b/toolz/tests/test_tlz.py @@ -2,6 +2,15 @@ def test_tlz(): + try: + import importlib + except ImportError: + try: + import tlz + 1/0 + except ImportError as e: + assert '"tlz" package is not available in Python 2.6' in str(e) + return import tlz tlz.curry tlz.functoolz.curry From a1e4048afecb986c9995363415cbe18823b67d8b Mon Sep 17 00:00:00 2001 From: Erik Welch Date: Sun, 12 Jun 2016 23:41:08 -0500 Subject: [PATCH 5/8] Copy docstrings too. --- tlz/__init__.py | 8 ++++++++ tlz/_build_tlz.py | 5 +++-- toolz/tests/test_tlz.py | 3 +++ 3 files changed, 14 insertions(+), 2 deletions(-) diff --git a/tlz/__init__.py b/tlz/__init__.py index 12707d9b..111c6051 100644 --- a/tlz/__init__.py +++ b/tlz/__init__.py @@ -1 +1,9 @@ +"""``tlz`` mirrors the ``toolz`` API and uses ``cytoolz`` if possible. + +The ``tlz`` package is installed when ``toolz`` is installed. It provides +a convenient way to use functions from ``cytoolz``--a faster Cython +implementation of ``toolz``--if it's installed, otherwise it uses functions +from ``toolz``. +""" + from . import _build_tlz diff --git a/tlz/_build_tlz.py b/tlz/_build_tlz.py index 52969b41..0227a7a9 100644 --- a/tlz/_build_tlz.py +++ b/tlz/_build_tlz.py @@ -51,8 +51,7 @@ def find_spec(self, fullname, path, target=None): # pragma: no cover return TlzSpec(fullname, self) def create_module(self, spec): - module = types.ModuleType(spec.name) - return module + return types.ModuleType(spec.name) def exec_module(self, module): toolz_mods = self._load_toolz(module.__name__) @@ -63,6 +62,8 @@ def exec_module(self, module): if package is not None: package, dot, submodules = package.partition('.') module.__package__ = ''.join(['tlz', dot, submodules]) + if not module.__doc__: + module.__doc__ = fast_mod.__doc__ # show file from toolz during introspection module.__file__ = slow_mod.__file__ diff --git a/toolz/tests/test_tlz.py b/toolz/tests/test_tlz.py index 865562e9..a77decf5 100644 --- a/toolz/tests/test_tlz.py +++ b/toolz/tests/test_tlz.py @@ -61,3 +61,6 @@ def test_tlz(): assert tlz.functoolz.__file__ == toolz.functoolz.__file__ assert tlz.pipe is toolz.pipe + + assert 'tlz' in tlz.__doc__ + assert tlz.curried.__doc__ is not None From 161200972f4724b39b3d3cc3781065a98c2b7639 Mon Sep 17 00:00:00 2001 From: Erik Welch Date: Sun, 12 Jun 2016 23:48:40 -0500 Subject: [PATCH 6/8] cleanup. --- tlz/__init__.py | 4 ++-- tlz/_build_tlz.py | 2 +- toolz/tests/test_tlz.py | 2 -- 3 files changed, 3 insertions(+), 5 deletions(-) diff --git a/tlz/__init__.py b/tlz/__init__.py index 111c6051..9c9c84af 100644 --- a/tlz/__init__.py +++ b/tlz/__init__.py @@ -2,8 +2,8 @@ The ``tlz`` package is installed when ``toolz`` is installed. It provides a convenient way to use functions from ``cytoolz``--a faster Cython -implementation of ``toolz``--if it's installed, otherwise it uses functions -from ``toolz``. +implementation of ``toolz``--if it is installed, otherwise it uses +functions from ``toolz``. """ from . import _build_tlz diff --git a/tlz/_build_tlz.py b/tlz/_build_tlz.py index 0227a7a9..b3a4b74e 100644 --- a/tlz/_build_tlz.py +++ b/tlz/_build_tlz.py @@ -8,7 +8,7 @@ class TlzLoader(object): - """ Finds and loads modules when added to sys.meta_path""" + """ Finds and loads ``tlz`` modules when added to sys.meta_path""" def __init__(self): self.always_from_toolz = set([ toolz.pipe, diff --git a/toolz/tests/test_tlz.py b/toolz/tests/test_tlz.py index a77decf5..68233e9d 100644 --- a/toolz/tests/test_tlz.py +++ b/toolz/tests/test_tlz.py @@ -17,7 +17,6 @@ def test_tlz(): assert tlz.__package__ == 'tlz' assert tlz.__name__ == 'tlz' import tlz.curried - print(tlz.curried.__package__) assert tlz.curried.__package__ == 'tlz.curried' assert tlz.curried.__name__ == 'tlz.curried' tlz.curried.curry @@ -51,7 +50,6 @@ def test_tlz(): try: import cytoolz assert cytoolz.__package__ == 'cytoolz' - #import cytoolz.curried assert cytoolz.curried.__package__ == 'cytoolz.curried' assert cytoolz.functoolz.__name__ == 'cytoolz.functoolz' except ImportError: From 05f2312409b00ebd67ccd95aebb115a4a00a4c26 Mon Sep 17 00:00:00 2001 From: Erik Welch Date: Sun, 30 Oct 2016 23:25:13 -0500 Subject: [PATCH 7/8] I was testing `tlz` in Python 2.6. I don't know the status of this. --- tlz/_build_tlz.py | 10 +++++----- toolz/compatibility.py | 7 +++++++ toolz/tests/test_tlz.py | 9 --------- 3 files changed, 12 insertions(+), 14 deletions(-) diff --git a/tlz/_build_tlz.py b/tlz/_build_tlz.py index b3a4b74e..9baa199e 100644 --- a/tlz/_build_tlz.py +++ b/tlz/_build_tlz.py @@ -1,11 +1,11 @@ import sys import types -try: - from importlib import import_module -except ImportError: - raise ImportError('"tlz" package is not available in Python 2.6') +#try: +# from importlib import import_module +#except ImportError: +# raise ImportError('"tlz" package is not available in Python 2.6') import toolz - +from toolz.compatibility import import_module class TlzLoader(object): """ Finds and loads ``tlz`` modules when added to sys.meta_path""" diff --git a/toolz/compatibility.py b/toolz/compatibility.py index 0fe40186..1f2a681e 100644 --- a/toolz/compatibility.py +++ b/toolz/compatibility.py @@ -31,3 +31,10 @@ iteritems = operator.methodcaller('iteritems') iterkeys = operator.methodcaller('iterkeys') itervalues = operator.methodcaller('itervalues') + +try: + from importlib import import_module +except ImportError: + def import_module(name): + __import__(name) + return sys.modules[name] diff --git a/toolz/tests/test_tlz.py b/toolz/tests/test_tlz.py index 68233e9d..ed2646a8 100644 --- a/toolz/tests/test_tlz.py +++ b/toolz/tests/test_tlz.py @@ -2,15 +2,6 @@ def test_tlz(): - try: - import importlib - except ImportError: - try: - import tlz - 1/0 - except ImportError as e: - assert '"tlz" package is not available in Python 2.6' in str(e) - return import tlz tlz.curry tlz.functoolz.curry From 05ede534aa0e30f9c99f9ba128c269697e24f5b0 Mon Sep 17 00:00:00 2001 From: Erik Welch Date: Thu, 3 Nov 2016 10:25:31 -0500 Subject: [PATCH 8/8] Support `tlz` in Python 2.6 using `toolz.compatibility.import_module` --- tlz/_build_tlz.py | 5 +---- toolz/tests/test_tlz.py | 9 --------- 2 files changed, 1 insertion(+), 13 deletions(-) diff --git a/tlz/_build_tlz.py b/tlz/_build_tlz.py index b3a4b74e..34eac604 100644 --- a/tlz/_build_tlz.py +++ b/tlz/_build_tlz.py @@ -1,10 +1,7 @@ import sys import types -try: - from importlib import import_module -except ImportError: - raise ImportError('"tlz" package is not available in Python 2.6') import toolz +from toolz.compatibility import import_module class TlzLoader(object): diff --git a/toolz/tests/test_tlz.py b/toolz/tests/test_tlz.py index 68233e9d..ed2646a8 100644 --- a/toolz/tests/test_tlz.py +++ b/toolz/tests/test_tlz.py @@ -2,15 +2,6 @@ def test_tlz(): - try: - import importlib - except ImportError: - try: - import tlz - 1/0 - except ImportError as e: - assert '"tlz" package is not available in Python 2.6' in str(e) - return import tlz tlz.curry tlz.functoolz.curry