From e221dc5fc5be90e9819453bba9ee621ea9193759 Mon Sep 17 00:00:00 2001 From: Arie Bovenberg Date: Wed, 16 May 2018 21:43:35 +0200 Subject: [PATCH 01/74] implement Compose.__repr__ --- toolz/functoolz.py | 4 ++++ toolz/tests/test_functoolz.py | 2 ++ 2 files changed, 6 insertions(+) diff --git a/toolz/functoolz.py b/toolz/functoolz.py index 408d73f3..3d8c2010 100644 --- a/toolz/functoolz.py +++ b/toolz/functoolz.py @@ -503,6 +503,10 @@ def __name__(self): except AttributeError: return type(self).__name__ + def __repr__(self): + return '{.__class__.__name__}({})'.format( + self, ', '.join(map(repr, reversed((self.first, ) + self.funcs)))) + def compose(*funcs): """ Compose functions to operate in series. diff --git a/toolz/tests/test_functoolz.py b/toolz/tests/test_functoolz.py index deda6068..417256ed 100644 --- a/toolz/tests/test_functoolz.py +++ b/toolz/tests/test_functoolz.py @@ -529,6 +529,8 @@ def g(a): assert composed.__name__ == 'Compose' assert composed.__doc__ == 'A composition of functions' + assert repr(composed) == 'Compose({!r}, {!r})'.format(f, h) + def test_pipe(): assert pipe(1, inc) == 2 From ba5066732ff735b04330c5dfc9aa804ee6f8ad0a Mon Sep 17 00:00:00 2001 From: Arie Bovenberg Date: Thu, 17 May 2018 08:13:14 +0200 Subject: [PATCH 02/74] implement Compose equality --- toolz/functoolz.py | 9 +++++++++ toolz/tests/test_functoolz.py | 32 ++++++++++++++++++++++++++++++++ 2 files changed, 41 insertions(+) diff --git a/toolz/functoolz.py b/toolz/functoolz.py index 3d8c2010..408eea88 100644 --- a/toolz/functoolz.py +++ b/toolz/functoolz.py @@ -507,6 +507,15 @@ def __repr__(self): return '{.__class__.__name__}({})'.format( self, ', '.join(map(repr, reversed((self.first, ) + self.funcs)))) + def __eq__(self, other): + if isinstance(other, Compose): + return other.first == self.first and other.funcs == self.funcs + return NotImplemented + + def __ne__(self, other): + equality = self.__eq__(other) + return NotImplemented if equality is NotImplemented else not equality + def compose(*funcs): """ Compose functions to operate in series. diff --git a/toolz/tests/test_functoolz.py b/toolz/tests/test_functoolz.py index 417256ed..213e830b 100644 --- a/toolz/tests/test_functoolz.py +++ b/toolz/tests/test_functoolz.py @@ -23,6 +23,26 @@ def double(x): return 2 * x +class AlwaysEquals(object): + """useful to test correct __eq__ implementation of other objects""" + + def __eq__(self, other): + return True + + def __ne__(self, other): + return False + + +class NeverEquals(object): + """useful to test correct __eq__ implementation of other objects""" + + def __eq__(self, other): + return False + + def __ne__(self, other): + return True + + def test_thread_first(): assert thread_first(2) == 2 assert thread_first(2, inc) == 3 @@ -531,6 +551,18 @@ def g(a): assert repr(composed) == 'Compose({!r}, {!r})'.format(f, h) + assert composed == compose(f, h) + assert composed == AlwaysEquals() + assert not composed == compose(h, f) + assert not composed == object() + assert not composed == NeverEquals() + + assert composed != compose(h, f) + assert composed != NeverEquals() + assert composed != object() + assert not composed != compose(f, h) + assert not composed != AlwaysEquals() + def test_pipe(): assert pipe(1, inc) == 2 From 3ded3fcda6b9388b937ace7b769ec1ecf2c002b8 Mon Sep 17 00:00:00 2001 From: Arie Bovenberg Date: Thu, 17 May 2018 08:14:12 +0200 Subject: [PATCH 03/74] refactor Compose.__repr__ --- toolz/functoolz.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/toolz/functoolz.py b/toolz/functoolz.py index 408eea88..32c51bdd 100644 --- a/toolz/functoolz.py +++ b/toolz/functoolz.py @@ -504,8 +504,8 @@ def __name__(self): return type(self).__name__ def __repr__(self): - return '{.__class__.__name__}({})'.format( - self, ', '.join(map(repr, reversed((self.first, ) + self.funcs)))) + return '{.__class__.__name__}{!r}'.format( + self, tuple(reversed((self.first, ) + self.funcs))) def __eq__(self, other): if isinstance(other, Compose): From d39d1ab8b86b08a47c69a66ca4aac4cdd836c42c Mon Sep 17 00:00:00 2001 From: Arie Bovenberg Date: Thu, 17 May 2018 08:26:42 +0200 Subject: [PATCH 04/74] implement compose.__hash__ --- toolz/functoolz.py | 3 +++ toolz/tests/test_functoolz.py | 3 +++ 2 files changed, 6 insertions(+) diff --git a/toolz/functoolz.py b/toolz/functoolz.py index 32c51bdd..ec097f56 100644 --- a/toolz/functoolz.py +++ b/toolz/functoolz.py @@ -516,6 +516,9 @@ def __ne__(self, other): equality = self.__eq__(other) return NotImplemented if equality is NotImplemented else not equality + def __hash__(self): + return hash(self.first) ^ hash(self.funcs) + def compose(*funcs): """ Compose functions to operate in series. diff --git a/toolz/tests/test_functoolz.py b/toolz/tests/test_functoolz.py index 213e830b..499a22e4 100644 --- a/toolz/tests/test_functoolz.py +++ b/toolz/tests/test_functoolz.py @@ -563,6 +563,9 @@ def g(a): assert not composed != compose(f, h) assert not composed != AlwaysEquals() + assert hash(composed) == hash(compose(f, h)) + assert hash(composed) != hash(compose(h, f)) + def test_pipe(): assert pipe(1, inc) == 2 From 9e878e1204158da905df76c241012a30146b8bc1 Mon Sep 17 00:00:00 2001 From: Arie Bovenberg Date: Thu, 17 May 2018 08:47:58 +0200 Subject: [PATCH 05/74] Compose now acts as method when bound to a class --- toolz/functoolz.py | 14 ++++++++++++++ toolz/tests/test_functoolz.py | 18 ++++++++++++++++++ 2 files changed, 32 insertions(+) diff --git a/toolz/functoolz.py b/toolz/functoolz.py index ec097f56..3d3be2be 100644 --- a/toolz/functoolz.py +++ b/toolz/functoolz.py @@ -3,6 +3,7 @@ import operator from operator import attrgetter from textwrap import dedent +from types import MethodType from .compatibility import PY3, PY33, PY34, PYPY, import_module from .utils import no_default @@ -519,6 +520,19 @@ def __ne__(self, other): def __hash__(self): return hash(self.first) ^ hash(self.funcs) + # Mimic the descriptor behavior of python functions. + # i.e. let Compose be called as a method when bound to a class. + if PY3: + # adapted from + # docs.python.org/3/howto/descriptor.html#functions-and-methods + def __get__(self, obj, objtype=None): + return self if obj is None else MethodType(self, obj) + else: + # adapted from + # docs.python.org/2/howto/descriptor.html#functions-and-methods + def __get__(self, obj, objtype=None): + return self if obj is None else MethodType(self, obj, objtype) + def compose(*funcs): """ Compose functions to operate in series. diff --git a/toolz/tests/test_functoolz.py b/toolz/tests/test_functoolz.py index 499a22e4..59534d46 100644 --- a/toolz/tests/test_functoolz.py +++ b/toolz/tests/test_functoolz.py @@ -566,6 +566,24 @@ def g(a): assert hash(composed) == hash(compose(f, h)) assert hash(composed) != hash(compose(h, f)) + bindable = compose(str, lambda x: x*2, lambda x, y=0: int(x) + y) + + class MyClass: + + def __int__(self): + return 8 + + my_method = bindable + my_static_method = staticmethod(bindable) + + assert MyClass.my_method(3) == '6' + assert MyClass.my_method(3, y=2) == '10' + assert MyClass.my_static_method(2) == '4' + assert MyClass().my_method() == '16' + assert MyClass().my_method(y=3) == '22' + assert MyClass().my_static_method(0) == '0' + assert MyClass().my_static_method(0, 1) == '2' + def test_pipe(): assert pipe(1, inc) == 2 From 5d95878603e759e4ee2a3149039cd0e2af07a329 Mon Sep 17 00:00:00 2001 From: Arie Bovenberg Date: Thu, 17 May 2018 19:31:37 +0200 Subject: [PATCH 06/74] implement Compose.__wrapped__ --- toolz/functoolz.py | 2 ++ toolz/tests/test_functoolz.py | 3 +++ 2 files changed, 5 insertions(+) diff --git a/toolz/functoolz.py b/toolz/functoolz.py index 3d3be2be..fc8c3801 100644 --- a/toolz/functoolz.py +++ b/toolz/functoolz.py @@ -533,6 +533,8 @@ def __get__(self, obj, objtype=None): def __get__(self, obj, objtype=None): return self if obj is None else MethodType(self, obj, objtype) + __wrapped__ = instanceproperty(attrgetter('first')) + def compose(*funcs): """ Compose functions to operate in series. diff --git a/toolz/tests/test_functoolz.py b/toolz/tests/test_functoolz.py index 59534d46..7b5c71ce 100644 --- a/toolz/tests/test_functoolz.py +++ b/toolz/tests/test_functoolz.py @@ -584,6 +584,9 @@ def __int__(self): assert MyClass().my_static_method(0) == '0' assert MyClass().my_static_method(0, 1) == '2' + assert compose(f, h).__wrapped__ is h + assert compose(f, h).__class__.__wrapped__ is None + def test_pipe(): assert pipe(1, inc) == 2 From ad9f297fbd600968523944b6b882523a8f76d743 Mon Sep 17 00:00:00 2001 From: Arie Bovenberg Date: Thu, 17 May 2018 19:51:39 +0200 Subject: [PATCH 07/74] implement basic Compose.__signature__ --- toolz/functoolz.py | 7 +++++++ toolz/tests/test_functoolz.py | 17 ++++++++++++++++- 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/toolz/functoolz.py b/toolz/functoolz.py index fc8c3801..3c491e12 100644 --- a/toolz/functoolz.py +++ b/toolz/functoolz.py @@ -533,6 +533,13 @@ def __get__(self, obj, objtype=None): def __get__(self, obj, objtype=None): return self if obj is None else MethodType(self, obj, objtype) + if PY3: + @instanceproperty + def __signature__(self): + base = inspect.signature(self.first) + last = inspect.signature(self.funcs[-1]) + return base.replace(return_annotation=last.return_annotation) + __wrapped__ = instanceproperty(attrgetter('first')) diff --git a/toolz/tests/test_functoolz.py b/toolz/tests/test_functoolz.py index 7b5c71ce..6cf425e7 100644 --- a/toolz/tests/test_functoolz.py +++ b/toolz/tests/test_functoolz.py @@ -1,7 +1,9 @@ +import inspect import platform from toolz.functoolz import (thread_first, thread_last, memoize, curry, compose, pipe, complement, do, juxt, flip, excepts) +from toolz.compatibility import PY3 from operator import add, mul, itemgetter from toolz.utils import raises from functools import partial @@ -587,6 +589,20 @@ def __int__(self): assert compose(f, h).__wrapped__ is h assert compose(f, h).__class__.__wrapped__ is None + # __signature__ is python3 only + if PY3: + + def myfunc(a: int, b: str, *c: float, d: int=4, **e: bool) -> int: + return 4 + + def otherfunc(f: int) -> str: + return 'result: {}'.format(f) + + composed = compose(otherfunc, myfunc) + sig = inspect.signature(composed) + assert sig.parameters == inspect.signature(myfunc).parameters + assert sig.return_annotation == str + def test_pipe(): assert pipe(1, inc) == 2 @@ -703,4 +719,3 @@ def raise_(a): excepting = excepts(object(), object(), object()) assert excepting.__name__ == 'excepting' assert excepting.__doc__ == excepts.__doc__ - From 8aaf970438fbd5e2c385dcae716af23c44ef67c3 Mon Sep 17 00:00:00 2001 From: Arie Bovenberg Date: Thu, 17 May 2018 20:00:03 +0200 Subject: [PATCH 08/74] extend Compose.__signature__ tests --- toolz/tests/test_functoolz.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/toolz/tests/test_functoolz.py b/toolz/tests/test_functoolz.py index 6cf425e7..6749048e 100644 --- a/toolz/tests/test_functoolz.py +++ b/toolz/tests/test_functoolz.py @@ -603,6 +603,18 @@ def otherfunc(f: int) -> str: assert sig.parameters == inspect.signature(myfunc).parameters assert sig.return_annotation == str + class MyClass: + method = composed + + assert len(inspect.signature(MyClass().method).parameters) == 4 + + try: + inspect.signature(compose(str, myfunc)) + except ValueError as e: + assert 'no signature' in e.args[0] + else: + raise AssertionError('expected exception was not raised') + def test_pipe(): assert pipe(1, inc) == 2 From 937de13862735b3bd597994d5532010955aa023c Mon Sep 17 00:00:00 2001 From: Arie Bovenberg Date: Thu, 17 May 2018 20:03:21 +0200 Subject: [PATCH 09/74] add clarifying comment --- toolz/functoolz.py | 1 + 1 file changed, 1 insertion(+) diff --git a/toolz/functoolz.py b/toolz/functoolz.py index 3c491e12..aee1f6a4 100644 --- a/toolz/functoolz.py +++ b/toolz/functoolz.py @@ -533,6 +533,7 @@ def __get__(self, obj, objtype=None): def __get__(self, obj, objtype=None): return self if obj is None else MethodType(self, obj, objtype) + # introspection with Signature is only possible from py3.3+ if PY3: @instanceproperty def __signature__(self): From c10bbbeef2277f41d5143a50d6dc9eddf331954f Mon Sep 17 00:00:00 2001 From: Arie Bovenberg Date: Wed, 23 May 2018 08:20:10 +0200 Subject: [PATCH 10/74] add coverage exemptions --- toolz/functoolz.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/toolz/functoolz.py b/toolz/functoolz.py index aee1f6a4..c843dc20 100644 --- a/toolz/functoolz.py +++ b/toolz/functoolz.py @@ -522,19 +522,19 @@ def __hash__(self): # Mimic the descriptor behavior of python functions. # i.e. let Compose be called as a method when bound to a class. - if PY3: + if PY3: # pragma: py2 no cover # adapted from # docs.python.org/3/howto/descriptor.html#functions-and-methods def __get__(self, obj, objtype=None): return self if obj is None else MethodType(self, obj) - else: + else: # pragma: py3 no cover # adapted from # docs.python.org/2/howto/descriptor.html#functions-and-methods def __get__(self, obj, objtype=None): return self if obj is None else MethodType(self, obj, objtype) # introspection with Signature is only possible from py3.3+ - if PY3: + if PY3: # pragma: py2 no cover @instanceproperty def __signature__(self): base = inspect.signature(self.first) From d7b8c40b4db4ebe3f6d1d488a9373b8a37fa187d Mon Sep 17 00:00:00 2001 From: Arie Bovenberg Date: Wed, 23 May 2018 08:47:33 +0200 Subject: [PATCH 11/74] set annotations in tests in python2-compatible manner --- toolz/tests/test_functoolz.py | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/toolz/tests/test_functoolz.py b/toolz/tests/test_functoolz.py index 6749048e..41dbc51c 100644 --- a/toolz/tests/test_functoolz.py +++ b/toolz/tests/test_functoolz.py @@ -592,12 +592,23 @@ def __int__(self): # __signature__ is python3 only if PY3: - def myfunc(a: int, b: str, *c: float, d: int=4, **e: bool) -> int: + def myfunc(a, b, c, *d, **e): return 4 - def otherfunc(f: int) -> str: + def otherfunc(f): return 'result: {}'.format(f) + # set annotations compatibly with python2 syntax + myfunc.__annotations__ = { + 'a': int, + 'b': str, + 'c': float, + 'd': int, + 'e': bool, + 'return': int, + } + otherfunc.__annotations__ = {'f': int, 'return': str} + composed = compose(otherfunc, myfunc) sig = inspect.signature(composed) assert sig.parameters == inspect.signature(myfunc).parameters From 3ef3de765f435d841026d66ea8cddf5b43be73fb Mon Sep 17 00:00:00 2001 From: Arie Bovenberg Date: Wed, 23 May 2018 08:47:56 +0200 Subject: [PATCH 12/74] remove signature exception propagation test. --- toolz/tests/test_functoolz.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/toolz/tests/test_functoolz.py b/toolz/tests/test_functoolz.py index 41dbc51c..f64ac9a3 100644 --- a/toolz/tests/test_functoolz.py +++ b/toolz/tests/test_functoolz.py @@ -619,13 +619,6 @@ class MyClass: assert len(inspect.signature(MyClass().method).parameters) == 4 - try: - inspect.signature(compose(str, myfunc)) - except ValueError as e: - assert 'no signature' in e.args[0] - else: - raise AssertionError('expected exception was not raised') - def test_pipe(): assert pipe(1, inc) == 2 From a33b57808f06d86d270713898ad07bf2739c30ed Mon Sep 17 00:00:00 2001 From: Ryan Grout Date: Tue, 5 Jun 2018 14:20:15 -0500 Subject: [PATCH 13/74] Avoid using .index() --- toolz/itertoolz.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/toolz/itertoolz.py b/toolz/itertoolz.py index a25eea3c..2ac2387e 100644 --- a/toolz/itertoolz.py +++ b/toolz/itertoolz.py @@ -720,7 +720,12 @@ def partition_all(n, seq): yield prev prev = item if prev[-1] is no_pad: - yield prev[:prev.index(no_pad)] + # Get first index of no_pad without using .index() + # https://github.com/pytoolz/toolz/issues/387 + for ind, item in enumerate(reversed(prev)): + if item is not no_pad: + yield prev[:len(prev)-ind] + break else: yield prev From 90ea648f9a42e9b84409df6895f1fb7507480ba1 Mon Sep 17 00:00:00 2001 From: Ryan Grout Date: Tue, 5 Jun 2018 16:13:02 -0500 Subject: [PATCH 14/74] Try to find first index of no_pad more intelligently. --- toolz/itertoolz.py | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/toolz/itertoolz.py b/toolz/itertoolz.py index 2ac2387e..d742dd68 100644 --- a/toolz/itertoolz.py +++ b/toolz/itertoolz.py @@ -720,12 +720,22 @@ def partition_all(n, seq): yield prev prev = item if prev[-1] is no_pad: - # Get first index of no_pad without using .index() - # https://github.com/pytoolz/toolz/issues/387 - for ind, item in enumerate(reversed(prev)): - if item is not no_pad: - yield prev[:len(prev)-ind] - break + try: + # If seq defines __len__, then we can quickly calculate where no_pad starts + yield prev[:len(seq) % n] + except TypeError: + # Get first index of no_pad without using .index() + # https://github.com/pytoolz/toolz/issues/387 + # We can employ modified binary search here to speed things up from O(n) to O(log n) + # Binary search from CPython's bisect module, modified for identity testing. + lo, hi = 0, len(prev) + while lo < hi: + mid = (lo + hi) // 2 + if prev[mid] is no_pad: + hi = mid + else: + lo = mid + 1 + yield prev[:lo] else: yield prev From c323698462133a56ac66af5c58cfb3fe65bcc53d Mon Sep 17 00:00:00 2001 From: Ryan Grout Date: Tue, 5 Jun 2018 16:36:58 -0500 Subject: [PATCH 15/74] len(prev) == n --- toolz/itertoolz.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/toolz/itertoolz.py b/toolz/itertoolz.py index d742dd68..6a3d4e22 100644 --- a/toolz/itertoolz.py +++ b/toolz/itertoolz.py @@ -728,7 +728,7 @@ def partition_all(n, seq): # https://github.com/pytoolz/toolz/issues/387 # We can employ modified binary search here to speed things up from O(n) to O(log n) # Binary search from CPython's bisect module, modified for identity testing. - lo, hi = 0, len(prev) + lo, hi = 0, n while lo < hi: mid = (lo + hi) // 2 if prev[mid] is no_pad: From 0b24bb1d044d6cc230c43210d91abf69e07a50a0 Mon Sep 17 00:00:00 2001 From: Ryan Grout Date: Tue, 5 Jun 2018 22:40:32 -0500 Subject: [PATCH 16/74] Add regression test. --- toolz/tests/test_itertoolz.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/toolz/tests/test_itertoolz.py b/toolz/tests/test_itertoolz.py index 93aa856d..02154e25 100644 --- a/toolz/tests/test_itertoolz.py +++ b/toolz/tests/test_itertoolz.py @@ -318,6 +318,16 @@ def test_partition_all(): assert list(partition_all(3, range(5))) == [(0, 1, 2), (3, 4)] assert list(partition_all(2, [])) == [] + # Regression test: https://github.com/pytoolz/toolz/issues/387 + class NoCompare(object): + def __eq__(self, other): + if self.__class__ == other.__class__: + return True + raise ValueError() + obj = NoCompare() + assert list(partition_all(2, [obj]*3)) == [(obj, obj), (obj,)] + + def test_count(): assert count((1, 2, 3)) == 3 From 03731458b385b2f445ee5e12ef6b7d5c46ae0eb9 Mon Sep 17 00:00:00 2001 From: Ryan Grout Date: Tue, 5 Jun 2018 22:43:22 -0500 Subject: [PATCH 17/74] Fix comments to make pep8 happy. --- toolz/itertoolz.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/toolz/itertoolz.py b/toolz/itertoolz.py index 6a3d4e22..71c7ddc0 100644 --- a/toolz/itertoolz.py +++ b/toolz/itertoolz.py @@ -721,13 +721,14 @@ def partition_all(n, seq): prev = item if prev[-1] is no_pad: try: - # If seq defines __len__, then we can quickly calculate where no_pad starts + # If seq defines __len__, then + # we can quickly calculate where no_pad starts yield prev[:len(seq) % n] except TypeError: # Get first index of no_pad without using .index() # https://github.com/pytoolz/toolz/issues/387 - # We can employ modified binary search here to speed things up from O(n) to O(log n) - # Binary search from CPython's bisect module, modified for identity testing. + # Binary search from CPython's bisect module, + # modified for identity testing. lo, hi = 0, n while lo < hi: mid = (lo + hi) // 2 From 5b0e57b94ad6c454444a9ac2f7fb1b12758e3521 Mon Sep 17 00:00:00 2001 From: Arie Bovenberg Date: Wed, 6 Jun 2018 07:45:16 +0200 Subject: [PATCH 18/74] drop py26 and py33 support --- .travis.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index c6e2098f..a07f83c4 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,9 +1,7 @@ sudo: false language: python python: - - "2.6" - "2.7" - - "3.3" - "3.4" - "3.5" - "3.6" From 7534eef2d6da0a55962feb156789a0d2d5ecf208 Mon Sep 17 00:00:00 2001 From: Ryan Grout Date: Wed, 6 Jun 2018 09:00:50 -0500 Subject: [PATCH 19/74] Remove added whitespace from tests. --- toolz/tests/test_itertoolz.py | 1 - 1 file changed, 1 deletion(-) diff --git a/toolz/tests/test_itertoolz.py b/toolz/tests/test_itertoolz.py index 02154e25..957e2143 100644 --- a/toolz/tests/test_itertoolz.py +++ b/toolz/tests/test_itertoolz.py @@ -328,7 +328,6 @@ def __eq__(self, other): assert list(partition_all(2, [obj]*3)) == [(obj, obj), (obj,)] - def test_count(): assert count((1, 2, 3)) == 3 assert count([]) == 0 From 86953ab1dfa252459182e81a9e0c6eaed9d1c686 Mon Sep 17 00:00:00 2001 From: Ryan Grout Date: Wed, 6 Jun 2018 09:13:29 -0500 Subject: [PATCH 20/74] Test both fast and slow paths and make sure they give same result. --- toolz/tests/test_itertoolz.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/toolz/tests/test_itertoolz.py b/toolz/tests/test_itertoolz.py index 957e2143..e0a8d17f 100644 --- a/toolz/tests/test_itertoolz.py +++ b/toolz/tests/test_itertoolz.py @@ -325,7 +325,9 @@ def __eq__(self, other): return True raise ValueError() obj = NoCompare() - assert list(partition_all(2, [obj]*3)) == [(obj, obj), (obj,)] + result = [(obj, obj, obj, obj), (obj, obj, obj)] + assert list(partition_all(4, [obj]*7)) == result + assert list(partition_all(4, iter([obj]*7))) == result def test_count(): From d1e8082097b3c11c3a166d43aab7f5846d8fa3fc Mon Sep 17 00:00:00 2001 From: Kale Franz Date: Fri, 6 Jul 2018 10:08:46 -0500 Subject: [PATCH 21/74] python3.7 generator expression fix Signed-off-by: Kale Franz --- toolz/functoolz.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/toolz/functoolz.py b/toolz/functoolz.py index 408d73f3..238d1b16 100644 --- a/toolz/functoolz.py +++ b/toolz/functoolz.py @@ -498,7 +498,7 @@ def composed_doc(*fs): def __name__(self): try: return '_of_'.join( - f.__name__ for f in reversed((self.first,) + self.funcs) + (f.__name__ for f in reversed((self.first,) + self.funcs)) ) except AttributeError: return type(self).__name__ From 267312681b4d9d73a22b692a7bdb9a2ddaf4a572 Mon Sep 17 00:00:00 2001 From: Floris Lambrechts Date: Tue, 10 Jul 2018 16:41:30 +0200 Subject: [PATCH 22/74] Point to itertoolz and functoolz at mrocklin's GitHub account These were deleted from the pytoolz GitHub account. --- doc/source/heritage.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/source/heritage.rst b/doc/source/heritage.rst index c66afea5..dcc92a0c 100644 --- a/doc/source/heritage.rst +++ b/doc/source/heritage.rst @@ -41,8 +41,8 @@ also exist within the Python ecosystem, most notably Fn.py_ and Funcy_. .. [itertools] http://docs.python.org/2/library/itertools.html .. [functools] http://docs.python.org/2/library/functools.html -.. [itertoolz] http://github.com/pytoolz/itertoolz -.. [functoolz] http://github.com/pytoolz/functoolz +.. [itertoolz] https://github.com/mrocklin/itertoolz +.. [functoolz] https://github.com/mrocklin/functoolz .. [Underscore.js] http://underscorejs.org .. [cheatsheet] http://clojure.org/cheatsheet .. [Guido] http://python-history.blogspot.com/2009/04/origins-of-pythons-functional-features.html From c27c178cf121291d11ba64dc5edefda105d309ee Mon Sep 17 00:00:00 2001 From: Floris Lambrechts Date: Tue, 10 Jul 2018 16:42:39 +0200 Subject: [PATCH 23/74] Change HTTP links to use HTTPS --- doc/source/heritage.rst | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/doc/source/heritage.rst b/doc/source/heritage.rst index dcc92a0c..f2fc587a 100644 --- a/doc/source/heritage.rst +++ b/doc/source/heritage.rst @@ -39,13 +39,13 @@ philosophy but mimics declarative database languages rather than functional ones. Enumerable_ is is the closest project in Ruby. Other excellent projects also exist within the Python ecosystem, most notably Fn.py_ and Funcy_. -.. [itertools] http://docs.python.org/2/library/itertools.html -.. [functools] http://docs.python.org/2/library/functools.html +.. [itertools] https://docs.python.org/2/library/itertools.html +.. [functools] https://docs.python.org/2/library/functools.html .. [itertoolz] https://github.com/mrocklin/itertoolz .. [functoolz] https://github.com/mrocklin/functoolz -.. [Underscore.js] http://underscorejs.org -.. [cheatsheet] http://clojure.org/cheatsheet -.. [Guido] http://python-history.blogspot.com/2009/04/origins-of-pythons-functional-features.html -.. [Enumerable] http://ruby-doc.org/core-2.0.0/Enumerable.html +.. [Underscore.js] https://underscorejs.org +.. [cheatsheet] https://clojure.org/cheatsheet +.. [Guido] https://python-history.blogspot.com/2009/04/origins-of-pythons-functional-features.html +.. [Enumerable] https://ruby-doc.org/core-2.0.0/Enumerable.html .. [funcy] https://github.com/suor/funcy/ .. [fn.py] https://github.com/kachayev/fn.py From 4ea6619556e02f218fa9b4c893d2d2aa41479c69 Mon Sep 17 00:00:00 2001 From: Ryan Grout Date: Thu, 9 Aug 2018 21:43:53 -0500 Subject: [PATCH 24/74] Avoid overhead of islice. --- toolz/itertoolz.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/toolz/itertoolz.py b/toolz/itertoolz.py index 71c7ddc0..bf91903f 100644 --- a/toolz/itertoolz.py +++ b/toolz/itertoolz.py @@ -374,7 +374,9 @@ def second(seq): >>> second('ABC') 'B' """ - return next(itertools.islice(seq, 1, None)) + seq = iter(seq) + next(seq) + return next(seq) def nth(n, seq): From a69f8a5c9d80b193fce84f5f126d09f4d3108338 Mon Sep 17 00:00:00 2001 From: Ryan Grout Date: Tue, 14 Aug 2018 09:20:05 -0500 Subject: [PATCH 25/74] Fix accumulate for Python 3.7 StopIteration is converted to RuntimeError if it escapes the generator frame. Code updated to reflect recommendations made in PEP479. --- toolz/itertoolz.py | 8 +++++++- toolz/tests/test_itertoolz.py | 1 + 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/toolz/itertoolz.py b/toolz/itertoolz.py index 71c7ddc0..429f6c30 100644 --- a/toolz/itertoolz.py +++ b/toolz/itertoolz.py @@ -56,7 +56,13 @@ def accumulate(binop, seq, initial=no_default): itertools.accumulate : In standard itertools for Python 3.2+ """ seq = iter(seq) - result = next(seq) if initial == no_default else initial + if initial == no_default: + try: + result = next(seq) + except StopIteration: + return + else: + result = initial yield result for elem in seq: result = binop(result, elem) diff --git a/toolz/tests/test_itertoolz.py b/toolz/tests/test_itertoolz.py index e0a8d17f..8a5c2407 100644 --- a/toolz/tests/test_itertoolz.py +++ b/toolz/tests/test_itertoolz.py @@ -289,6 +289,7 @@ def binop(a, b): start = object() assert list(accumulate(binop, [], start)) == [start] + assert list(accumulate(binop, [])) == [] assert list(accumulate(add, [1, 2, 3], no_default2)) == [1, 3, 6] From 701712f8947aa845b01eb90505104885dbc95a26 Mon Sep 17 00:00:00 2001 From: Ryan Grout Date: Tue, 14 Aug 2018 21:44:59 -0500 Subject: [PATCH 26/74] Handle StopIteration in Python 3.7+ Add more tests for edge cases --- toolz/itertoolz.py | 7 +++++-- toolz/tests/test_itertoolz.py | 8 ++++++++ 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/toolz/itertoolz.py b/toolz/itertoolz.py index bf91903f..b8028c49 100644 --- a/toolz/itertoolz.py +++ b/toolz/itertoolz.py @@ -375,8 +375,11 @@ def second(seq): 'B' """ seq = iter(seq) - next(seq) - return next(seq) + try: + next(seq) + return next(seq) + except StopIteration: + raise def nth(n, seq): diff --git a/toolz/tests/test_itertoolz.py b/toolz/tests/test_itertoolz.py index e0a8d17f..bd83db2f 100644 --- a/toolz/tests/test_itertoolz.py +++ b/toolz/tests/test_itertoolz.py @@ -17,6 +17,8 @@ from toolz.compatibility import range, filter from operator import add, mul +import pytest + # is comparison will fail between this and no_default no_default2 = loads(dumps('__no__default__')) @@ -143,6 +145,12 @@ def test_second(): assert second('ABCDE') == 'B' assert second((3, 2, 1)) == 2 assert isinstance(second({0: 'zero', 1: 'one'}), int) + assert second(x for x in range(2)) == 1 + + # Python 3.7, StopIteration should be raised if iterable too short + with pytest.raises(StopIteration): + second([]) + second([0]) def test_last(): From 307b0bbdc1d7247b9d86ea3abf5b1f89648b9f60 Mon Sep 17 00:00:00 2001 From: Ryan Grout Date: Tue, 14 Aug 2018 22:14:54 -0500 Subject: [PATCH 27/74] Revert "Handle StopIteration in Python 3.7+" This reverts commit 701712f8947aa845b01eb90505104885dbc95a26. --- toolz/itertoolz.py | 7 ++----- toolz/tests/test_itertoolz.py | 8 -------- 2 files changed, 2 insertions(+), 13 deletions(-) diff --git a/toolz/itertoolz.py b/toolz/itertoolz.py index b8028c49..bf91903f 100644 --- a/toolz/itertoolz.py +++ b/toolz/itertoolz.py @@ -375,11 +375,8 @@ def second(seq): 'B' """ seq = iter(seq) - try: - next(seq) - return next(seq) - except StopIteration: - raise + next(seq) + return next(seq) def nth(n, seq): diff --git a/toolz/tests/test_itertoolz.py b/toolz/tests/test_itertoolz.py index bd83db2f..e0a8d17f 100644 --- a/toolz/tests/test_itertoolz.py +++ b/toolz/tests/test_itertoolz.py @@ -17,8 +17,6 @@ from toolz.compatibility import range, filter from operator import add, mul -import pytest - # is comparison will fail between this and no_default no_default2 = loads(dumps('__no__default__')) @@ -145,12 +143,6 @@ def test_second(): assert second('ABCDE') == 'B' assert second((3, 2, 1)) == 2 assert isinstance(second({0: 'zero', 1: 'one'}), int) - assert second(x for x in range(2)) == 1 - - # Python 3.7, StopIteration should be raised if iterable too short - with pytest.raises(StopIteration): - second([]) - second([0]) def test_last(): From 5b222b35f1875048c0940fa27733e216c64a2e34 Mon Sep 17 00:00:00 2001 From: Elias Mistler Date: Thu, 16 Aug 2018 14:05:55 +0100 Subject: [PATCH 28/74] added clojure style function --- toolz/functoolz.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/toolz/functoolz.py b/toolz/functoolz.py index 238d1b16..e8bfc3ae 100644 --- a/toolz/functoolz.py +++ b/toolz/functoolz.py @@ -21,6 +21,19 @@ def identity(x): return x +def apply(func, *args, **kwargs): + """ Applies a function and returns the results + >>> def double(x): return 2*x + >>> def inc(x): return x + 1 + >>> apply(double, 5) + 10 + + >>> tuple(map(apply, [double, inc, double], [10, 500, 8000])) + (20, 501, 16000) + """ + return func(*args, **kwargs) + + def thread_first(val, *forms): """ Thread value through a sequence of functions/forms From 89665afc3f92cc67c6ea2da337729c31e1145d37 Mon Sep 17 00:00:00 2001 From: Elias Mistler Date: Thu, 16 Aug 2018 14:26:54 +0100 Subject: [PATCH 29/74] test for apply --- toolz/functoolz.py | 2 +- toolz/tests/test_functoolz.py | 7 ++++++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/toolz/functoolz.py b/toolz/functoolz.py index e8bfc3ae..792ddbc5 100644 --- a/toolz/functoolz.py +++ b/toolz/functoolz.py @@ -8,7 +8,7 @@ from .utils import no_default -__all__ = ('identity', 'thread_first', 'thread_last', 'memoize', 'compose', +__all__ = ('identity', 'apply', 'thread_first', 'thread_last', 'memoize', 'compose', 'pipe', 'complement', 'juxt', 'do', 'curry', 'flip', 'excepts') diff --git a/toolz/tests/test_functoolz.py b/toolz/tests/test_functoolz.py index deda6068..01830a9c 100644 --- a/toolz/tests/test_functoolz.py +++ b/toolz/tests/test_functoolz.py @@ -1,7 +1,7 @@ import platform from toolz.functoolz import (thread_first, thread_last, memoize, curry, - compose, pipe, complement, do, juxt, flip, excepts) + compose, pipe, complement, do, juxt, flip, excepts, apply) from operator import add, mul, itemgetter from toolz.utils import raises from functools import partial @@ -23,6 +23,11 @@ def double(x): return 2 * x +def test_apply(): + assert apply(double, 5) == 10 + assert tuple(map(apply, [double, inc, double], [10, 500, 8000])) == (20, 501, 16000) + + def test_thread_first(): assert thread_first(2) == 2 assert thread_first(2, inc) == 3 From eb2603027aaa017b558b5946cf4e3911be3c87dc Mon Sep 17 00:00:00 2001 From: Ryan Grout Date: Thu, 23 Aug 2018 22:17:51 -0500 Subject: [PATCH 30/74] More efficient assoc. --- toolz/dicttoolz.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/toolz/dicttoolz.py b/toolz/dicttoolz.py index f840e7fc..a5f0044a 100644 --- a/toolz/dicttoolz.py +++ b/toolz/dicttoolz.py @@ -192,8 +192,9 @@ def assoc(d, key, value, factory=dict): {'x': 1, 'y': 3} """ d2 = factory() + d2.update(d) d2[key] = value - return merge(d, d2, factory=factory) + return d2 def dissoc(d, *keys): From b791293abe5ea29a588b71c8cde289914b120796 Mon Sep 17 00:00:00 2001 From: Hugo Date: Thu, 27 Sep 2018 10:39:08 +0300 Subject: [PATCH 31/74] Remove broken download badge And update some links to HTTPS and fix some typos. --- README.rst | 22 ++++++++++------------ 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/README.rst b/README.rst index ee1628c7..a1a31e2b 100644 --- a/README.rst +++ b/README.rst @@ -1,7 +1,7 @@ Toolz ===== -|Build Status| |Coverage Status| |Version Status| |Downloads| +|Build Status| |Coverage Status| |Version Status| A set of utility functions for iterators, functions, and dictionaries. @@ -30,7 +30,7 @@ Structure and Heritage ``unique``, ``interpose``, |literal functoolz|_, for higher-order functions. Examples: ``memoize``, -``curry``, ``compose`` +``curry``, ``compose``, |literal dicttoolz|_, for operations on dictionaries. Examples: ``assoc``, ``update-in``, ``merge``. @@ -77,7 +77,7 @@ Dependencies It is pure Python and requires no dependencies beyond the standard library. -It is, in short, a light weight dependency. +It is, in short, a lightweight dependency. CyToolz @@ -86,21 +86,21 @@ CyToolz The ``toolz`` project has been reimplemented in `Cython `__. The ``cytoolz`` project is a drop-in replacement for the Pure Python implementation. -See `CyToolz Github Page `__ for more +See `CyToolz GitHub Page `__ for more details. See Also -------- -- `Underscore.js `__: A similar library for +- `Underscore.js `__: A similar library for JavaScript -- `Enumerable `__: A +- `Enumerable `__: A similar library for Ruby -- `Clojure `__: A functional language whose +- `Clojure `__: A functional language whose standard library has several counterparts in ``toolz`` -- `itertools `__: The +- `itertools `__: The Python standard library for iterator tools -- `functools `__: The +- `functools `__: The Python standard library for function tools Contributions Welcome @@ -129,6 +129,4 @@ We're friendly. .. |Coverage Status| image:: https://coveralls.io/repos/pytoolz/toolz/badge.svg?branch=master :target: https://coveralls.io/r/pytoolz/toolz .. |Version Status| image:: https://badge.fury.io/py/toolz.svg - :target: http://badge.fury.io/py/toolz -.. |Downloads| image:: https://img.shields.io/pypi/dm/toolz.svg - :target: https://pypi.python.org/pypi/toolz/ + :target: https://badge.fury.io/py/toolz From bb7ff0baa8398e1f06e1d84c1c9d827bbee1d68a Mon Sep 17 00:00:00 2001 From: Hugo Date: Thu, 27 Sep 2018 10:44:50 +0300 Subject: [PATCH 32/74] Add support for Python 3.7 --- .travis.yml | 11 +++++++---- setup.py | 1 + 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index c6e2098f..f9546b3e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -10,13 +10,16 @@ python: - "3.7-dev" - "pypy" +# Enable 3.7 without globally enabling sudo and dist: xenial for other build jobs +matrix: + include: + - python: 3.7 + dist: xenial + sudo: true + env: - PEP8_IGNORE="E731,W503,E402" -matrix: - allow_failures: - - python: "3.7-dev" - # command to install dependencies install: - pip install coverage pep8 pytest diff --git a/setup.py b/setup.py index 7ae76e18..bce2cd03 100755 --- a/setup.py +++ b/setup.py @@ -32,5 +32,6 @@ "Programming Language :: Python :: 3.4", "Programming Language :: Python :: 3.5", "Programming Language :: Python :: 3.6", + "Programming Language :: Python :: 3.7", "Programming Language :: Python :: Implementation :: CPython", "Programming Language :: Python :: Implementation :: PyPy"]) From 76b6b28dc8919b55f9f33ed32a4d61bb3d125465 Mon Sep 17 00:00:00 2001 From: Hugo Date: Thu, 27 Sep 2018 10:59:21 +0300 Subject: [PATCH 33/74] Drop support for EOL Python 2.6 and 3.3 --- .binstar.yml | 2 -- .travis.yml | 2 -- README.rst | 2 +- doc/source/install.rst | 2 +- setup.py | 3 +-- tlz/_build_tlz.py | 2 +- toolz/_signatures.py | 3 ++- toolz/compatibility.py | 10 +--------- toolz/functoolz.py | 8 ++------ toolz/tests/test_curried.py | 2 +- toolz/tests/test_inspect_args.py | 4 ++-- toolz/tests/test_serialization.py | 8 ++++---- 12 files changed, 16 insertions(+), 32 deletions(-) diff --git a/.binstar.yml b/.binstar.yml index 43606529..86ea8b8d 100644 --- a/.binstar.yml +++ b/.binstar.yml @@ -6,9 +6,7 @@ platform: - win-64 - win-32 engine: - - python=2.6 - python=2.7 - - python=3.3 - python=3.4 script: - conda build conda.recipe diff --git a/.travis.yml b/.travis.yml index c6e2098f..a07f83c4 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,9 +1,7 @@ sudo: false language: python python: - - "2.6" - "2.7" - - "3.3" - "3.4" - "3.5" - "3.6" diff --git a/README.rst b/README.rst index ee1628c7..0e65e7fc 100644 --- a/README.rst +++ b/README.rst @@ -73,7 +73,7 @@ This builds a standard wordcount function from pieces within ``toolz``: Dependencies ------------ -``toolz`` supports Python 2.6+ and Python 3.3+ with a common codebase. +``toolz`` supports Python 2.7 and Python 3.4+ with a common codebase. It is pure Python and requires no dependencies beyond the standard library. diff --git a/doc/source/install.rst b/doc/source/install.rst index 07051b17..cfba80aa 100644 --- a/doc/source/install.rst +++ b/doc/source/install.rst @@ -11,4 +11,4 @@ three ways: 1. Toolz is pure Python 2. Toolz relies only on the standard library -3. Toolz simultaneously supports Python versions 2.6, 2.7, 3.3, 3.4, 3.5, 3.6, PyPy +3. Toolz simultaneously supports Python versions 2.7, 3.4+, PyPy diff --git a/setup.py b/setup.py index 7ae76e18..b622b498 100755 --- a/setup.py +++ b/setup.py @@ -25,10 +25,9 @@ "Development Status :: 5 - Production/Stable", "License :: OSI Approved :: BSD License", "Programming Language :: Python", - "Programming Language :: Python :: 2.6", + "Programming Language :: Python :: 2", "Programming Language :: Python :: 2.7", "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.3", "Programming Language :: Python :: 3.4", "Programming Language :: Python :: 3.5", "Programming Language :: Python :: 3.6", diff --git a/tlz/_build_tlz.py b/tlz/_build_tlz.py index 34eac604..0c6e33aa 100644 --- a/tlz/_build_tlz.py +++ b/tlz/_build_tlz.py @@ -1,7 +1,7 @@ import sys import types import toolz -from toolz.compatibility import import_module +from importlib import import_module class TlzLoader(object): diff --git a/toolz/_signatures.py b/toolz/_signatures.py index 28aa95ce..3f512e84 100644 --- a/toolz/_signatures.py +++ b/toolz/_signatures.py @@ -16,8 +16,9 @@ import inspect import itertools import operator +from importlib import import_module -from .compatibility import PY3, import_module +from .compatibility import PY3 from .functoolz import (is_partial_args, is_arity, has_varargs, has_keywords, num_required_args) diff --git a/toolz/compatibility.py b/toolz/compatibility.py index 67635026..b8d1d049 100644 --- a/toolz/compatibility.py +++ b/toolz/compatibility.py @@ -1,13 +1,12 @@ import operator import sys PY3 = sys.version_info[0] > 2 -PY33 = sys.version_info[0] == 3 and sys.version_info[1] == 3 PY34 = sys.version_info[0] == 3 and sys.version_info[1] == 4 PYPY = hasattr(sys, 'pypy_version_info') __all__ = ('map', 'filter', 'range', 'zip', 'reduce', 'zip_longest', 'iteritems', 'iterkeys', 'itervalues', 'filterfalse', - 'PY3', 'PY34', 'PYPY', 'import_module') + 'PY3', 'PY34', 'PYPY') if PY3: map = map @@ -31,10 +30,3 @@ iteritems = operator.methodcaller('iteritems') iterkeys = operator.methodcaller('iterkeys') itervalues = operator.methodcaller('itervalues') - -try: - from importlib import import_module -except ImportError: - def import_module(name): - __import__(name) - return sys.modules[name] diff --git a/toolz/functoolz.py b/toolz/functoolz.py index 238d1b16..5c8aecb6 100644 --- a/toolz/functoolz.py +++ b/toolz/functoolz.py @@ -2,9 +2,10 @@ import inspect import operator from operator import attrgetter +from importlib import import_module from textwrap import dedent -from .compatibility import PY3, PY33, PY34, PYPY, import_module +from .compatibility import PY3, PY34, PYPY from .utils import no_default @@ -758,11 +759,6 @@ def _check_sigspec(sigspec, func, builtin_func, *builtin_args): and (( hasattr(func, '__signature__') and hasattr(func.__signature__, '__get__') - ) or ( - PY33 - and hasattr(func, '__wrapped__') - and hasattr(func.__wrapped__, '__get__') - and not callable(func.__wrapped__) )) ): # pragma: no cover (not covered in Python 3.4) val = builtin_func(*builtin_args) diff --git a/toolz/tests/test_curried.py b/toolz/tests/test_curried.py index 2efb0001..47d99832 100644 --- a/toolz/tests/test_curried.py +++ b/toolz/tests/test_curried.py @@ -2,8 +2,8 @@ import toolz.curried from toolz.curried import (take, first, second, sorted, merge_with, reduce, merge, operator as cop) -from toolz.compatibility import import_module from collections import defaultdict +from importlib import import_module from operator import add diff --git a/toolz/tests/test_inspect_args.py b/toolz/tests/test_inspect_args.py index ef03331e..74c1f03c 100644 --- a/toolz/tests/test_inspect_args.py +++ b/toolz/tests/test_inspect_args.py @@ -7,7 +7,7 @@ num_required_args, has_varargs, has_keywords) from toolz._signatures import builtins import toolz._signatures as _sigs -from toolz.compatibility import PY3, PY33 +from toolz.compatibility import PY3 from toolz.utils import raises @@ -495,6 +495,6 @@ def __wrapped__(self): if PY3: assert inspect.signature(func) == inspect.signature(wrapped) - assert num_required_args(Wrapped) == (False if PY33 else None) + assert num_required_args(Wrapped) == None _sigs.signatures[Wrapped] = (_sigs.expand_sig((0, lambda func: None)),) assert num_required_args(Wrapped) == 1 diff --git a/toolz/tests/test_serialization.py b/toolz/tests/test_serialization.py index 1780c508..1b35bfd6 100644 --- a/toolz/tests/test_serialization.py +++ b/toolz/tests/test_serialization.py @@ -2,7 +2,7 @@ import toolz import toolz.curried import pickle -from toolz.compatibility import PY3, PY33, PY34 +from toolz.compatibility import PY3 from toolz.utils import raises @@ -148,7 +148,7 @@ def preserves_identity(obj): # If we add `curry.__getattr__` forwarding, the following tests will pass - # if not PY33 and not PY34: + # if not PY34: # assert preserves_identity(GlobalCurried.func.g1) # assert preserves_identity(GlobalCurried.func.NestedCurried.func.g2) # assert preserves_identity(GlobalCurried.func.Nested) @@ -159,7 +159,7 @@ def preserves_identity(obj): # assert preserves_identity(GlobalCurried.NestedCurried) # assert preserves_identity(GlobalCurried.NestedCurried.f2) # assert preserves_identity(GlobalCurried.Nested.f3) - # if not PY33 and not PY34: + # if not PY34: # assert preserves_identity(GlobalCurried.g1) # assert preserves_identity(GlobalCurried.NestedCurried.g2) # assert preserves_identity(GlobalCurried.Nested) @@ -175,7 +175,7 @@ def preserves_identity(obj): # assert func1 is not func2 # assert func1(4) == func2(4) == 10 # - # if not PY33 and not PY34: + # if not PY34: # nested3 = GlobalCurried.func.Nested(1, 2) # nested4 = pickle.loads(pickle.dumps(nested3)) # assert nested3 is not nested4 From 04855e5e22820e107c33074a16beb0cf61560dd9 Mon Sep 17 00:00:00 2001 From: Hugo Date: Thu, 27 Sep 2018 11:06:45 +0300 Subject: [PATCH 34/74] Use 'is None' rather than equality comparison --- toolz/tests/test_inspect_args.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/toolz/tests/test_inspect_args.py b/toolz/tests/test_inspect_args.py index 74c1f03c..98a83e9e 100644 --- a/toolz/tests/test_inspect_args.py +++ b/toolz/tests/test_inspect_args.py @@ -495,6 +495,6 @@ def __wrapped__(self): if PY3: assert inspect.signature(func) == inspect.signature(wrapped) - assert num_required_args(Wrapped) == None + assert num_required_args(Wrapped) is None _sigs.signatures[Wrapped] = (_sigs.expand_sig((0, lambda func: None)),) assert num_required_args(Wrapped) == 1 From ddd428ac461621e32c70b46c7c6baed4bb5e1e8c Mon Sep 17 00:00:00 2001 From: Hugo Date: Thu, 27 Sep 2018 11:11:25 +0300 Subject: [PATCH 35/74] Remove redundant parentheses --- toolz/_signatures.py | 2 +- toolz/functoolz.py | 2 +- toolz/itertoolz.py | 2 +- toolz/tests/test_itertoolz.py | 26 +++++++++++++------------- toolz/tests/test_serialization.py | 4 ++-- 5 files changed, 18 insertions(+), 18 deletions(-) diff --git a/toolz/_signatures.py b/toolz/_signatures.py index 3f512e84..d2ad70fc 100644 --- a/toolz/_signatures.py +++ b/toolz/_signatures.py @@ -698,7 +698,7 @@ def expand_sig(sig): num_pos_only = num_pos_args(sigspec) keyword_only = () keyword_exclude = get_exclude_keywords(num_pos_only, sigspec) - return (num_pos_only, func, keyword_only + keyword_exclude, sigspec) + return num_pos_only, func, keyword_only + keyword_exclude, sigspec signatures = {} diff --git a/toolz/functoolz.py b/toolz/functoolz.py index 5c8aecb6..36d2cf1a 100644 --- a/toolz/functoolz.py +++ b/toolz/functoolz.py @@ -352,7 +352,7 @@ def __reduce__(self): if k not in ('_partial', '_sigspec')) state = (type(self), func, self.args, self.keywords, userdict, is_decorated) - return (_restore_curry, state) + return _restore_curry, state def _restore_curry(cls, func, args, kwargs, userdict, is_decorated): diff --git a/toolz/itertoolz.py b/toolz/itertoolz.py index 941b9a9f..9d615c00 100644 --- a/toolz/itertoolz.py +++ b/toolz/itertoolz.py @@ -457,7 +457,7 @@ def get(ind, seq, default=no_default): if len(ind) > 1: return operator.itemgetter(*ind)(seq) elif ind: - return (seq[ind[0]],) + return seq[ind[0]], else: return () else: diff --git a/toolz/tests/test_itertoolz.py b/toolz/tests/test_itertoolz.py index 8a5c2407..c7097b9b 100644 --- a/toolz/tests/test_itertoolz.py +++ b/toolz/tests/test_itertoolz.py @@ -368,10 +368,10 @@ def addpair(pair): result = set(starmap(add, join(first, names, second, fruit))) - expected = set([((1, 'one', 'apple', 1)), - ((1, 'one', 'orange', 1)), - ((2, 'two', 'banana', 2)), - ((2, 'two', 'coconut', 2))]) + expected = set([(1, 'one', 'apple', 1), + (1, 'one', 'orange', 1), + (2, 'two', 'banana', 2), + (2, 'two', 'coconut', 2)]) assert result == expected @@ -409,14 +409,14 @@ def test_join_double_repeats(): result = set(starmap(add, join(first, names, second, fruit))) - expected = set([((1, 'one', 'apple', 1)), - ((1, 'one', 'orange', 1)), - ((2, 'two', 'banana', 2)), - ((2, 'two', 'coconut', 2)), - ((1, 'uno', 'apple', 1)), - ((1, 'uno', 'orange', 1)), - ((2, 'dos', 'banana', 2)), - ((2, 'dos', 'coconut', 2))]) + expected = set([(1, 'one', 'apple', 1), + (1, 'one', 'orange', 1), + (2, 'two', 'banana', 2), + (2, 'two', 'coconut', 2), + (1, 'uno', 'apple', 1), + (1, 'uno', 'orange', 1), + (2, 'dos', 'banana', 2), + (2, 'dos', 'coconut', 2)]) assert result == expected @@ -427,7 +427,7 @@ def test_join_missing_element(): result = set(starmap(add, join(first, names, second, fruit))) - expected = set([((1, 'one', 'orange', 1))]) + expected = set([(1, 'one', 'orange', 1)]) assert result == expected diff --git a/toolz/tests/test_serialization.py b/toolz/tests/test_serialization.py index 1b35bfd6..92b07f05 100644 --- a/toolz/tests/test_serialization.py +++ b/toolz/tests/test_serialization.py @@ -81,7 +81,7 @@ def g1(self): def __reduce__(self): """Allow us to serialize instances of GlobalCurried""" - return (GlobalCurried, (self.x, self.y)) + return GlobalCurried, (self.x, self.y) @toolz.curry class NestedCurried(object): @@ -98,7 +98,7 @@ def g2(self): def __reduce__(self): """Allow us to serialize instances of NestedCurried""" - return (GlobalCurried.NestedCurried, (self.x, self.y)) + return GlobalCurried.NestedCurried, (self.x, self.y) class Nested(object): def __init__(self, x, y): From 7fe24c99ff0988be05a7688f5561e63dff51f26f Mon Sep 17 00:00:00 2001 From: Hugo Date: Thu, 27 Sep 2018 11:24:17 +0300 Subject: [PATCH 36/74] Add python_requires to help pip --- setup.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index b622b498..a5b5166c 100755 --- a/setup.py +++ b/setup.py @@ -7,7 +7,7 @@ setup(name='toolz', version=toolz.__version__, description='List processing tools and functional utilities', - url='http://github.com/pytoolz/toolz/', + url='https://github.com/pytoolz/toolz/', author='https://raw.github.com/pytoolz/toolz/master/AUTHORS.md', maintainer='Matthew Rocklin', maintainer_email='mrocklin@gmail.com', @@ -21,6 +21,7 @@ long_description=(open('README.rst').read() if exists('README.rst') else ''), zip_safe=False, + python_requires=">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*", classifiers=[ "Development Status :: 5 - Production/Stable", "License :: OSI Approved :: BSD License", From 8851fcc9c71aec05a76691087b786a197548bf67 Mon Sep 17 00:00:00 2001 From: Hugo Date: Sat, 6 Oct 2018 10:04:37 +0300 Subject: [PATCH 37/74] Upgrade Python syntax with pyupgrade (but not % formatting) --- tlz/_build_tlz.py | 4 ++-- toolz/curried/operator.py | 4 ++-- toolz/dicttoolz.py | 4 ++-- toolz/sandbox/tests/test_core.py | 4 ++-- toolz/sandbox/tests/test_parallel.py | 4 ++-- toolz/tests/test_curried.py | 8 ++++---- toolz/tests/test_functoolz.py | 2 +- toolz/tests/test_inspect_args.py | 2 +- toolz/tests/test_itertoolz.py | 18 +++++++++--------- 9 files changed, 25 insertions(+), 25 deletions(-) diff --git a/tlz/_build_tlz.py b/tlz/_build_tlz.py index 0c6e33aa..3c017a54 100644 --- a/tlz/_build_tlz.py +++ b/tlz/_build_tlz.py @@ -7,9 +7,9 @@ class TlzLoader(object): """ Finds and loads ``tlz`` modules when added to sys.meta_path""" def __init__(self): - self.always_from_toolz = set([ + self.always_from_toolz = { toolz.pipe, - ]) + } def _load_toolz(self, fullname): rv = {} diff --git a/toolz/curried/operator.py b/toolz/curried/operator.py index 6af55e5d..8bc9e523 100644 --- a/toolz/curried/operator.py +++ b/toolz/curried/operator.py @@ -11,8 +11,8 @@ def should_curry(f): locals().update( - dict((name, curry(f) if should_curry(f) else f) - for name, f in vars(operator).items() if callable(f)), + {name: curry(f) if should_curry(f) else f + for name, f in vars(operator).items() if callable(f)}, ) # Clean up the namespace. diff --git a/toolz/dicttoolz.py b/toolz/dicttoolz.py index f840e7fc..b81dce8f 100644 --- a/toolz/dicttoolz.py +++ b/toolz/dicttoolz.py @@ -11,8 +11,8 @@ def _get_factory(f, kwargs): factory = kwargs.pop('factory', dict) if kwargs: - raise TypeError("{0}() got an unexpected keyword argument " - "'{1}'".format(f.__name__, kwargs.popitem()[0])) + raise TypeError("{}() got an unexpected keyword argument " + "'{}'".format(f.__name__, kwargs.popitem()[0])) return factory diff --git a/toolz/sandbox/tests/test_core.py b/toolz/sandbox/tests/test_core.py index 96a83094..14e3847f 100644 --- a/toolz/sandbox/tests/test_core.py +++ b/toolz/sandbox/tests/test_core.py @@ -20,8 +20,8 @@ def test_EqualityHashKey_default_key(): data2 = list(map(EqualityHashDefault, [T0, T0, T1, T1, (), (1,)])) data2.extend([T0, T1, (), (1,)]) set3 = set(data2) - assert set3 == set([(), (1,), EqualityHashDefault(()), - EqualityHashDefault((1,))]) + assert set3 == {(), (1,), EqualityHashDefault(()), + EqualityHashDefault((1,))} assert len(set3) == 4 assert EqualityHashDefault(()) in set3 assert EqualityHashDefault((1,)) in set3 diff --git a/toolz/sandbox/tests/test_parallel.py b/toolz/sandbox/tests/test_parallel.py index e645dfdd..e22c3de2 100644 --- a/toolz/sandbox/tests/test_parallel.py +++ b/toolz/sandbox/tests/test_parallel.py @@ -17,8 +17,8 @@ def setadd(s, item): s.add(item) return s - assert fold(setadd, [1, 2, 3], set()) == set((1, 2, 3)) + assert fold(setadd, [1, 2, 3], set()) == {1, 2, 3} assert (fold(setadd, [1, 2, 3], set(), chunksize=2, combine=set.union) - == set((1, 2, 3))) + == {1, 2, 3}) assert fold(add, range(10), default=no_default2) == fold(add, range(10)) diff --git a/toolz/tests/test_curried.py b/toolz/tests/test_curried.py index 47d99832..5db4d079 100644 --- a/toolz/tests/test_curried.py +++ b/toolz/tests/test_curried.py @@ -62,7 +62,7 @@ def test_curried_operator(): ) # Make sure this isn't totally empty. - assert len(set(vars(cop)) & set(['add', 'sub', 'mul'])) == 3 + assert len(set(vars(cop)) & {'add', 'sub', 'mul'}) == 3 def test_curried_namespace(): @@ -79,10 +79,10 @@ def should_curry(func): def curry_namespace(ns): - return dict( - (name, toolz.curry(f) if should_curry(f) else f) + return { + name: toolz.curry(f) if should_curry(f) else f for name, f in ns.items() if '__' not in name - ) + } from_toolz = curry_namespace(vars(toolz)) from_exceptions = curry_namespace(vars(exceptions)) diff --git a/toolz/tests/test_functoolz.py b/toolz/tests/test_functoolz.py index deda6068..29c43851 100644 --- a/toolz/tests/test_functoolz.py +++ b/toolz/tests/test_functoolz.py @@ -328,7 +328,7 @@ def bar(a, b, c=1): b1 = curry(bar, 1, c=2) assert b1 != f1 - assert set([f1, f2, g1, h1, h2, h3, b1, b1()]) == set([f1, g1, h1, b1]) + assert {f1, f2, g1, h1, h2, h3, b1, b1()} == {f1, g1, h1, b1} # test unhashable input unhash1 = curry(foo, []) diff --git a/toolz/tests/test_inspect_args.py b/toolz/tests/test_inspect_args.py index 98a83e9e..77457ed6 100644 --- a/toolz/tests/test_inspect_args.py +++ b/toolz/tests/test_inspect_args.py @@ -437,7 +437,7 @@ def is_missing(modname, name, func): if missing: messages = [] for modname, names in sorted(missing.items()): - msg = '{0}:\n {1}'.format(modname, '\n '.join(sorted(names))) + msg = '{}:\n {}'.format(modname, '\n '.join(sorted(names))) messages.append(msg) message = 'Missing introspection for the following callables:\n\n' raise AssertionError(message + '\n\n'.join(messages)) diff --git a/toolz/tests/test_itertoolz.py b/toolz/tests/test_itertoolz.py index c7097b9b..3526cb2d 100644 --- a/toolz/tests/test_itertoolz.py +++ b/toolz/tests/test_itertoolz.py @@ -271,7 +271,7 @@ def set_add(s, i): return s assert reduceby(iseven, set_add, [1, 2, 3, 4, 1, 2], set) == \ - {True: set([2, 4]), False: set([1, 3])} + {True: {2, 4}, False: {1, 3}} def test_iterate(): @@ -368,10 +368,10 @@ def addpair(pair): result = set(starmap(add, join(first, names, second, fruit))) - expected = set([(1, 'one', 'apple', 1), + expected = {(1, 'one', 'apple', 1), (1, 'one', 'orange', 1), (2, 'two', 'banana', 2), - (2, 'two', 'coconut', 2)]) + (2, 'two', 'coconut', 2)} assert result == expected @@ -409,14 +409,14 @@ def test_join_double_repeats(): result = set(starmap(add, join(first, names, second, fruit))) - expected = set([(1, 'one', 'apple', 1), + expected = {(1, 'one', 'apple', 1), (1, 'one', 'orange', 1), (2, 'two', 'banana', 2), (2, 'two', 'coconut', 2), (1, 'uno', 'apple', 1), (1, 'uno', 'orange', 1), (2, 'dos', 'banana', 2), - (2, 'dos', 'coconut', 2)]) + (2, 'dos', 'coconut', 2)} assert result == expected @@ -427,21 +427,21 @@ def test_join_missing_element(): result = set(starmap(add, join(first, names, second, fruit))) - expected = set([(1, 'one', 'orange', 1)]) + expected = {(1, 'one', 'orange', 1)} assert result == expected def test_left_outer_join(): result = set(join(identity, [1, 2], identity, [2, 3], left_default=None)) - expected = set([(2, 2), (None, 3)]) + expected = {(2, 2), (None, 3)} assert result == expected def test_right_outer_join(): result = set(join(identity, [1, 2], identity, [2, 3], right_default=None)) - expected = set([(2, 2), (1, None)]) + expected = {(2, 2), (1, None)} assert result == expected @@ -449,7 +449,7 @@ def test_right_outer_join(): def test_outer_join(): result = set(join(identity, [1, 2], identity, [2, 3], left_default=None, right_default=None)) - expected = set([(2, 2), (1, None), (None, 3)]) + expected = {(2, 2), (1, None), (None, 3)} assert result == expected From a6665b1ea02c266c4412d1a310a300994e953409 Mon Sep 17 00:00:00 2001 From: Ryan Grout Date: Thu, 18 Oct 2018 16:04:07 -0500 Subject: [PATCH 38/74] Make dissoc smarter. Dissoc was really fast when the keys to remove were relatively few compared to the size of the dictionary. This version will pick the smaller container to iterate over, either building up a new dictionary or tearing down a copy. A small heuristic is used to decide if the overhead of building a set is worth the effort. My rudimentry testing shows that if the number of keys is less than 60% the length of the dictionary, tearing down a copy is faster. If keys is very large, we calculate the intersection between keys and mapping and build a new dictionary out of that intersection. --- toolz/dicttoolz.py | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/toolz/dicttoolz.py b/toolz/dicttoolz.py index a5f0044a..9efc2dfb 100644 --- a/toolz/dicttoolz.py +++ b/toolz/dicttoolz.py @@ -197,7 +197,7 @@ def assoc(d, key, value, factory=dict): return d2 -def dissoc(d, *keys): +def dissoc(d, *keys, **kwargs): """ Return a new dict with the given key(s) removed. New dict has d[key] deleted for each supplied key. @@ -210,11 +210,20 @@ def dissoc(d, *keys): >>> dissoc({'x': 1}, 'y') # Ignores missing keys {'x': 1} """ - d2 = copy.copy(d) - for key in keys: - if key in d2: - del d2[key] - return d2 + factory = _get_factory(dissoc, kwargs) + d2 = factory() + + if len(keys) < len(d) * .6: + d2.update(d) + for key in keys: + if key in d2: + del d2[key] + else: + remaining = set(d) + remaining.difference_update(keys) + for k in remaining: + d2[k] = d[k] + return d2 def assoc_in(d, keys, value, factory=dict): From d55666d6eb9d19b963d4be6d3cd8c2edd7533547 Mon Sep 17 00:00:00 2001 From: Ryan Grout Date: Thu, 18 Oct 2018 16:12:27 -0500 Subject: [PATCH 39/74] Remove import. --- toolz/dicttoolz.py | 1 - 1 file changed, 1 deletion(-) diff --git a/toolz/dicttoolz.py b/toolz/dicttoolz.py index 9efc2dfb..ce2e53e3 100644 --- a/toolz/dicttoolz.py +++ b/toolz/dicttoolz.py @@ -1,4 +1,3 @@ -import copy import operator from toolz.compatibility import (map, zip, iteritems, iterkeys, itervalues, reduce) From 03cf714895ea317b84e888344d784b1fa4e167d1 Mon Sep 17 00:00:00 2001 From: Ryan Grout Date: Thu, 18 Oct 2018 16:18:12 -0500 Subject: [PATCH 40/74] Fix indentation error. --- toolz/dicttoolz.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/toolz/dicttoolz.py b/toolz/dicttoolz.py index ce2e53e3..7ede1f62 100644 --- a/toolz/dicttoolz.py +++ b/toolz/dicttoolz.py @@ -222,7 +222,7 @@ def dissoc(d, *keys, **kwargs): remaining.difference_update(keys) for k in remaining: d2[k] = d[k] - return d2 + return d2 def assoc_in(d, keys, value, factory=dict): From c6a2779023958b268105cd6e37d358d5a5563d00 Mon Sep 17 00:00:00 2001 From: Ryan Grout Date: Thu, 18 Oct 2018 22:39:47 -0500 Subject: [PATCH 41/74] Add dissoc to curried namespace. Looks like it might have been forgotten. --- toolz/curried/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/toolz/curried/__init__.py b/toolz/curried/__init__.py index 43aeffd4..c9c90bbb 100644 --- a/toolz/curried/__init__.py +++ b/toolz/curried/__init__.py @@ -59,6 +59,7 @@ assoc_in = toolz.curry(toolz.assoc_in) cons = toolz.curry(toolz.cons) countby = toolz.curry(toolz.countby) +dissoc = toolz.curry(toolz.dissoc) do = toolz.curry(toolz.do) drop = toolz.curry(toolz.drop) excepts = toolz.curry(toolz.excepts) From 18947feed97e4a3287e5a56656a1dbb6186090c8 Mon Sep 17 00:00:00 2001 From: Ryan Grout Date: Fri, 2 Nov 2018 23:26:10 -0500 Subject: [PATCH 42/74] Avoid error catching inside loop for significant performance gain. --- toolz/itertoolz.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/toolz/itertoolz.py b/toolz/itertoolz.py index 941b9a9f..77099d4c 100644 --- a/toolz/itertoolz.py +++ b/toolz/itertoolz.py @@ -875,13 +875,11 @@ def join(leftkey, leftseq, rightkey, rightseq, for item in rightseq: key = rightkey(item) seen_keys.add(key) - try: - left_matches = d[key] - for match in left_matches: - yield (match, item) - except KeyError: - if not left_default_is_no_default: - yield (left_default, item) + if key in d: + for left_match in d[key]: + yield (left_match, item) + elif not left_default_is_no_default: + yield (left_default, item) if right_default != no_default: for key, matches in d.items(): From afd126fb6861725b4f00380a81e61e9a154bb681 Mon Sep 17 00:00:00 2001 From: Ryan Grout Date: Fri, 2 Nov 2018 23:28:24 -0500 Subject: [PATCH 43/74] On Python 2, don't create a copy of the dictionary items. --- toolz/itertoolz.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/toolz/itertoolz.py b/toolz/itertoolz.py index 77099d4c..45031c33 100644 --- a/toolz/itertoolz.py +++ b/toolz/itertoolz.py @@ -882,7 +882,7 @@ def join(leftkey, leftseq, rightkey, rightseq, yield (left_default, item) if right_default != no_default: - for key, matches in d.items(): + for key, matches in iteritems(d): if key not in seen_keys: for match in matches: yield (match, right_default) From 33c6687c13d91f7b1ec09ec12e9d5dd314c77f2c Mon Sep 17 00:00:00 2001 From: Ryan Grout Date: Wed, 7 Nov 2018 16:48:46 -0600 Subject: [PATCH 44/74] Detect join case and perform each case separately. Added notes about memory usage and hashability. --- toolz/itertoolz.py | 58 +++++++++++++++++++++++++++++++++++----------- 1 file changed, 44 insertions(+), 14 deletions(-) diff --git a/toolz/itertoolz.py b/toolz/itertoolz.py index 45031c33..7fed9d26 100644 --- a/toolz/itertoolz.py +++ b/toolz/itertoolz.py @@ -816,6 +816,8 @@ def join(leftkey, leftseq, rightkey, rightseq, This is a semi-streaming operation. The LEFT sequence is fully evaluated and placed into memory. The RIGHT sequence is evaluated lazily and so can be arbitrarily large. + (Note: If right_default is defined, then unique keys of rightseq + will also be stored in memory.) >>> friends = [('Alice', 'Edith'), ... ('Alice', 'Zhao'), @@ -858,7 +860,9 @@ def join(leftkey, leftseq, rightkey, rightseq, Usually the key arguments are callables to be applied to the sequences. If the keys are not obviously callable then it is assumed that indexing was - intended, e.g. the following is a legal change + intended, e.g. the following is a legal change. + The join is implemented as a hash join and the keys of leftseq must be hashable. + Additionally, if right_default is defined, then keys of rightseq must also be hashable. >>> # result = join(second, friends, first, cities) >>> result = join(1, friends, 0, cities) # doctest: +SKIP @@ -869,19 +873,45 @@ def join(leftkey, leftseq, rightkey, rightseq, rightkey = getter(rightkey) d = groupby(leftkey, leftseq) - seen_keys = set() - - left_default_is_no_default = (left_default == no_default) - for item in rightseq: - key = rightkey(item) - seen_keys.add(key) - if key in d: - for left_match in d[key]: - yield (left_match, item) - elif not left_default_is_no_default: - yield (left_default, item) - - if right_default != no_default: + + if (left_default is no_default) and (right_default is no_default): + # Inner Join + for item in rightseq: + key = rightkey(item) + if key in d: + for left_match in d[key]: + yield (left_match, item) + elif (left_default is not no_default) and (right_default is no_default): + # Right Join + for item in rightseq: + key = rightkey(item) + if key in d: + for left_match in d[key]: + yield (left_match, item) + else: + yield (left_default, item) + elif (right_default is not no_default): + seen_keys = set() + + if left_default is no_default: + # Left Join + for item in rightseq: + key = rightkey(item) + seen_keys.add(key) + if key in d: + for left_match in d[key]: + yield(left_match, item) + else: + # Full Join + for item in rightseq: + key = rightkey(item) + seen_keys.add(key) + if key in d: + for left_match in d[key]: + yield (left_match, item) + else: + yield (left_default, item) + for key, matches in iteritems(d): if key not in seen_keys: for match in matches: From a99299e62ba5bf0d3522293bce34bbd6fd9223ee Mon Sep 17 00:00:00 2001 From: Ryan Grout Date: Wed, 7 Nov 2018 23:12:08 -0600 Subject: [PATCH 45/74] Restore identity comparisons. --- toolz/itertoolz.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/toolz/itertoolz.py b/toolz/itertoolz.py index 7fed9d26..280180e4 100644 --- a/toolz/itertoolz.py +++ b/toolz/itertoolz.py @@ -874,14 +874,14 @@ def join(leftkey, leftseq, rightkey, rightseq, d = groupby(leftkey, leftseq) - if (left_default is no_default) and (right_default is no_default): + if (left_default == no_default) and (right_default == no_default): # Inner Join for item in rightseq: key = rightkey(item) if key in d: for left_match in d[key]: yield (left_match, item) - elif (left_default is not no_default) and (right_default is no_default): + elif (left_default != no_default) and (right_default == no_default): # Right Join for item in rightseq: key = rightkey(item) @@ -890,17 +890,17 @@ def join(leftkey, leftseq, rightkey, rightseq, yield (left_match, item) else: yield (left_default, item) - elif (right_default is not no_default): + elif (right_default != no_default): seen_keys = set() - if left_default is no_default: + if left_default == no_default: # Left Join for item in rightseq: key = rightkey(item) seen_keys.add(key) if key in d: for left_match in d[key]: - yield(left_match, item) + yield (left_match, item) else: # Full Join for item in rightseq: From 9ffd119674a09c2a5cd5ee9b042574872aec6ed6 Mon Sep 17 00:00:00 2001 From: Ryan Grout Date: Wed, 7 Nov 2018 23:15:22 -0600 Subject: [PATCH 46/74] Adjust line breaks to 79 characters. --- toolz/itertoolz.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/toolz/itertoolz.py b/toolz/itertoolz.py index 280180e4..bf72f94c 100644 --- a/toolz/itertoolz.py +++ b/toolz/itertoolz.py @@ -861,8 +861,9 @@ def join(leftkey, leftseq, rightkey, rightseq, Usually the key arguments are callables to be applied to the sequences. If the keys are not obviously callable then it is assumed that indexing was intended, e.g. the following is a legal change. - The join is implemented as a hash join and the keys of leftseq must be hashable. - Additionally, if right_default is defined, then keys of rightseq must also be hashable. + The join is implemented as a hash join and the keys of leftseq must be + hashable. Additionally, if right_default is defined, then keys of rightseq + must also be hashable. >>> # result = join(second, friends, first, cities) >>> result = join(1, friends, 0, cities) # doctest: +SKIP From 01e4313cb9a272be7a535b98dc7f69f56c3ecdca Mon Sep 17 00:00:00 2001 From: Hugo Date: Sat, 6 Oct 2018 10:23:50 +0300 Subject: [PATCH 47/74] Fix DeprecationWarning in Python 3.7 --- toolz/itertoolz.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/toolz/itertoolz.py b/toolz/itertoolz.py index 9d615c00..d1b7f933 100644 --- a/toolz/itertoolz.py +++ b/toolz/itertoolz.py @@ -8,6 +8,13 @@ filter) from toolz.utils import no_default +try: + # Python 3 + from collections.abc import Sequence +except ImportError: + # Python 2.7 + from collections import Sequence + __all__ = ('remove', 'accumulate', 'groupby', 'merge_sorted', 'interleave', 'unique', 'isiterable', 'isdistinct', 'take', 'drop', 'take_nth', @@ -391,7 +398,7 @@ def nth(n, seq): >>> nth(1, 'ABC') 'B' """ - if isinstance(seq, (tuple, list, collections.Sequence)): + if isinstance(seq, (tuple, list, Sequence)): return seq[n] else: return next(itertools.islice(seq, n, None)) From cb94a5bd258e2e0c7a2ebbab169ba5804ee781b6 Mon Sep 17 00:00:00 2001 From: Hugo Date: Sat, 6 Oct 2018 10:24:25 +0300 Subject: [PATCH 48/74] Ignore file created when running tests --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 3d4bfdf9..1b611177 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,4 @@ build/ dist/ *.egg-info/ +bench/shakespeare.txt From e9cea93ae1d828c6ec807aa6551404f8f433ac43 Mon Sep 17 00:00:00 2001 From: Hugo Date: Sat, 6 Oct 2018 10:43:00 +0300 Subject: [PATCH 49/74] Don't cover non-runnable code on Python 3 --- toolz/itertoolz.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/toolz/itertoolz.py b/toolz/itertoolz.py index d1b7f933..b72bd991 100644 --- a/toolz/itertoolz.py +++ b/toolz/itertoolz.py @@ -11,7 +11,7 @@ try: # Python 3 from collections.abc import Sequence -except ImportError: +except ImportError: # pragma: no cover # Python 2.7 from collections import Sequence From a055df371563fa7d4cab634fa511fce0561cf541 Mon Sep 17 00:00:00 2001 From: Hugo Date: Sat, 6 Oct 2018 10:43:31 +0300 Subject: [PATCH 50/74] Ignore file created when running tests with coverage --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 1b611177..6e6f4b7d 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,4 @@ build/ dist/ *.egg-info/ bench/shakespeare.txt +.coverage From 1804af0a2ea7968239786229b47152a85ae75087 Mon Sep 17 00:00:00 2001 From: Hugo Date: Tue, 13 Nov 2018 11:06:23 +0200 Subject: [PATCH 51/74] Move compat code to compatibility.py --- toolz/compatibility.py | 2 ++ toolz/itertoolz.py | 9 +-------- 2 files changed, 3 insertions(+), 8 deletions(-) diff --git a/toolz/compatibility.py b/toolz/compatibility.py index b8d1d049..51e3673f 100644 --- a/toolz/compatibility.py +++ b/toolz/compatibility.py @@ -19,6 +19,7 @@ iteritems = operator.methodcaller('items') iterkeys = operator.methodcaller('keys') itervalues = operator.methodcaller('values') + from collections.abc import Sequence else: range = xrange reduce = reduce @@ -30,3 +31,4 @@ iteritems = operator.methodcaller('iteritems') iterkeys = operator.methodcaller('iterkeys') itervalues = operator.methodcaller('itervalues') + from collections import Sequence diff --git a/toolz/itertoolz.py b/toolz/itertoolz.py index b72bd991..a063c4c7 100644 --- a/toolz/itertoolz.py +++ b/toolz/itertoolz.py @@ -5,16 +5,9 @@ from functools import partial from random import Random from toolz.compatibility import (map, filterfalse, zip, zip_longest, iteritems, - filter) + filter, Sequence) from toolz.utils import no_default -try: - # Python 3 - from collections.abc import Sequence -except ImportError: # pragma: no cover - # Python 2.7 - from collections import Sequence - __all__ = ('remove', 'accumulate', 'groupby', 'merge_sorted', 'interleave', 'unique', 'isiterable', 'isdistinct', 'take', 'drop', 'take_nth', From 68a3591f33305d5458c9e72bff015589c7c06d75 Mon Sep 17 00:00:00 2001 From: Elias Mistler Date: Fri, 23 Nov 2018 12:13:24 +0000 Subject: [PATCH 52/74] change signature of `apply` as per @eriknw --- toolz/functoolz.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/toolz/functoolz.py b/toolz/functoolz.py index 792ddbc5..78eccff2 100644 --- a/toolz/functoolz.py +++ b/toolz/functoolz.py @@ -21,7 +21,7 @@ def identity(x): return x -def apply(func, *args, **kwargs): +def apply(*func_and_args, **kwargs): """ Applies a function and returns the results >>> def double(x): return 2*x >>> def inc(x): return x + 1 @@ -31,6 +31,9 @@ def apply(func, *args, **kwargs): >>> tuple(map(apply, [double, inc, double], [10, 500, 8000])) (20, 501, 16000) """ + if not func_and_args: + raise TypeError('func argument is required') + func, args = func_and_args[0], func_and_args[1:] return func(*args, **kwargs) From dcebedde4b5c8ceeffca1a2ae42a35bf61d53e0b Mon Sep 17 00:00:00 2001 From: Elias Mistler Date: Fri, 23 Nov 2018 12:23:21 +0000 Subject: [PATCH 53/74] add apply to toolz.curried --- toolz/curried/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/toolz/curried/__init__.py b/toolz/curried/__init__.py index 43aeffd4..11a0d32f 100644 --- a/toolz/curried/__init__.py +++ b/toolz/curried/__init__.py @@ -55,6 +55,7 @@ from .exceptions import merge, merge_with accumulate = toolz.curry(toolz.accumulate) +apply = toolz.curry(toolz.apply) assoc = toolz.curry(toolz.assoc) assoc_in = toolz.curry(toolz.assoc_in) cons = toolz.curry(toolz.cons) From 4d670925aca373cca4bb450b39022f487e101b91 Mon Sep 17 00:00:00 2001 From: Elias Mistler Date: Fri, 23 Nov 2018 13:30:55 +0000 Subject: [PATCH 54/74] add apply to toolz.curried --- toolz/curried/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/toolz/curried/__init__.py b/toolz/curried/__init__.py index 11a0d32f..fab6255f 100644 --- a/toolz/curried/__init__.py +++ b/toolz/curried/__init__.py @@ -26,6 +26,7 @@ import toolz from . import operator from toolz import ( + apply, comp, complement, compose, @@ -55,7 +56,6 @@ from .exceptions import merge, merge_with accumulate = toolz.curry(toolz.accumulate) -apply = toolz.curry(toolz.apply) assoc = toolz.curry(toolz.assoc) assoc_in = toolz.curry(toolz.assoc_in) cons = toolz.curry(toolz.cons) From 65c81ca35a46596f27798fe2a7ba94ef845b904f Mon Sep 17 00:00:00 2001 From: Elias Mistler Date: Fri, 23 Nov 2018 13:36:12 +0000 Subject: [PATCH 55/74] format __all__ to fit in line breaks --- toolz/functoolz.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/toolz/functoolz.py b/toolz/functoolz.py index 78eccff2..be6fa204 100644 --- a/toolz/functoolz.py +++ b/toolz/functoolz.py @@ -8,8 +8,9 @@ from .utils import no_default -__all__ = ('identity', 'apply', 'thread_first', 'thread_last', 'memoize', 'compose', - 'pipe', 'complement', 'juxt', 'do', 'curry', 'flip', 'excepts') +__all__ = ('identity', 'apply', 'thread_first', 'thread_last', 'memoize', + 'compose', 'pipe', 'complement', 'juxt', 'do', 'curry', 'flip', + 'excepts') def identity(x): From 47047483fb30b58ef765adaf01cc03777ac56375 Mon Sep 17 00:00:00 2001 From: Elias Mistler Date: Fri, 23 Nov 2018 13:46:08 +0000 Subject: [PATCH 56/74] complete code coverage --- toolz/tests/test_functoolz.py | 1 + 1 file changed, 1 insertion(+) diff --git a/toolz/tests/test_functoolz.py b/toolz/tests/test_functoolz.py index 01830a9c..dcb43af1 100644 --- a/toolz/tests/test_functoolz.py +++ b/toolz/tests/test_functoolz.py @@ -26,6 +26,7 @@ def double(x): def test_apply(): assert apply(double, 5) == 10 assert tuple(map(apply, [double, inc, double], [10, 500, 8000])) == (20, 501, 16000) + assert raises(TypeError, apply) def test_thread_first(): From 62a686966fe7dae3e6c7e1a0ca1248f8b46c72cc Mon Sep 17 00:00:00 2001 From: lumbric Date: Fri, 30 Nov 2018 16:30:20 +0100 Subject: [PATCH 57/74] Add hint for groupby to documentation Same name, but does something different. Might be confusing, so let's warn users explicitly. --- toolz/itertoolz.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/toolz/itertoolz.py b/toolz/itertoolz.py index 9d615c00..81ada6f8 100644 --- a/toolz/itertoolz.py +++ b/toolz/itertoolz.py @@ -89,6 +89,8 @@ def groupby(key, seq): 'M': [{'gender': 'M', 'name': 'Bob'}, {'gender': 'M', 'name': 'Charlie'}]} + Not to be confused with ``itertools.groupby`` + See Also: countby """ From 60192346df93e5efcdbfb53fa287a963f558c3f6 Mon Sep 17 00:00:00 2001 From: lumbric Date: Fri, 30 Nov 2018 16:47:52 +0100 Subject: [PATCH 58/74] Fix wrong docstring for unzip() The current version of unzip() works only if the inner sequence is finite, but in Python 3 zip() can be used. See also comment in #239. --- toolz/sandbox/core.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/toolz/sandbox/core.py b/toolz/sandbox/core.py index 359fc3fa..915f06c2 100644 --- a/toolz/sandbox/core.py +++ b/toolz/sandbox/core.py @@ -105,7 +105,7 @@ def unzip(seq): [1, 2] Unlike the naive implementation ``def unzip(seq): zip(*seq)`` this - implementation can handle a finite sequence of infinite sequences. + implementation can handle an infinite sequence ``seq``. Caveats: @@ -113,7 +113,8 @@ def unzip(seq): of auxiliary storage if the resulting iterators are consumed at different times. - * The top level sequence cannot be infinite. + * The inner sequence cannot be infinite. In Python 3 ``zip(*seq)`` can be + used if ``seq`` is a finite sequence of infinite sequences. """ From ba9d148b4170e4080e09128de99426c49c74810f Mon Sep 17 00:00:00 2001 From: Ryan Grout Date: Wed, 5 Dec 2018 21:22:31 -0600 Subject: [PATCH 59/74] Use a non-recursive implementation inspired by cytoolz. --- toolz/dicttoolz.py | 31 +++++++++++++++++++++---------- 1 file changed, 21 insertions(+), 10 deletions(-) diff --git a/toolz/dicttoolz.py b/toolz/dicttoolz.py index e07b35cd..84ba3a01 100644 --- a/toolz/dicttoolz.py +++ b/toolz/dicttoolz.py @@ -7,7 +7,6 @@ 'valfilter', 'keyfilter', 'itemfilter', 'assoc', 'dissoc', 'assoc_in', 'update_in', 'get_in') - def _get_factory(f, kwargs): factory = kwargs.pop('factory', dict) if kwargs: @@ -266,16 +265,28 @@ def update_in(d, keys, func, default=None, factory=dict): >>> update_in({1: 'foo'}, [2, 3, 4], inc, 0) {1: 'foo', 2: {3: {4: 1}}} """ - assert len(keys) > 0 - k, ks = keys[0], keys[1:] - if ks: - return assoc(d, k, update_in(d[k] if (k in d) else factory(), - ks, func, default, factory), - factory) - else: - innermost = func(d[k]) if (k in d) else func(default) - return assoc(d, k, innermost, factory) + ks = iter(keys) + k = next(ks) + + rv = inner = factory() + rv.update(d) + + for key in ks: + if k in d: + d = d[k] + dtemp = factory() + dtemp.update(d) + else: + d = dtemp = factory() + inner[k] = inner = dtemp + k = key + + if k in d: + inner[k] = func(d[k]) + else: + inner[k] = func(default) + return rv def get_in(keys, coll, default=None, no_default=False): """ Returns coll[i0][i1]...[iX] where [i0, i1, ..., iX]==keys. From 0217f38a2fde4e9bd2cbf51de0cbe38a29f9ff7b Mon Sep 17 00:00:00 2001 From: Ryan Grout Date: Wed, 5 Dec 2018 21:55:52 -0600 Subject: [PATCH 60/74] Remove unnecessary parens. --- toolz/itertoolz.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/toolz/itertoolz.py b/toolz/itertoolz.py index bf72f94c..80d1fe5f 100644 --- a/toolz/itertoolz.py +++ b/toolz/itertoolz.py @@ -875,14 +875,14 @@ def join(leftkey, leftseq, rightkey, rightseq, d = groupby(leftkey, leftseq) - if (left_default == no_default) and (right_default == no_default): + if left_default == no_default and right_default == no_default: # Inner Join for item in rightseq: key = rightkey(item) if key in d: for left_match in d[key]: yield (left_match, item) - elif (left_default != no_default) and (right_default == no_default): + elif left_default != no_default and right_default == no_default: # Right Join for item in rightseq: key = rightkey(item) @@ -891,7 +891,7 @@ def join(leftkey, leftseq, rightkey, rightseq, yield (left_match, item) else: yield (left_default, item) - elif (right_default != no_default): + elif right_default != no_default: seen_keys = set() if left_default == no_default: From 54aa44292e433a6f7829c383f0fb9cbf5c4f09b3 Mon Sep 17 00:00:00 2001 From: Ryan Grout Date: Wed, 5 Dec 2018 22:02:49 -0600 Subject: [PATCH 61/74] Avoid lookups for set.add method. --- toolz/itertoolz.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/toolz/itertoolz.py b/toolz/itertoolz.py index 80d1fe5f..e07bb765 100644 --- a/toolz/itertoolz.py +++ b/toolz/itertoolz.py @@ -893,12 +893,13 @@ def join(leftkey, leftseq, rightkey, rightseq, yield (left_default, item) elif right_default != no_default: seen_keys = set() + seen = seen_keys.add if left_default == no_default: # Left Join for item in rightseq: key = rightkey(item) - seen_keys.add(key) + seen(key) if key in d: for left_match in d[key]: yield (left_match, item) @@ -906,7 +907,7 @@ def join(leftkey, leftseq, rightkey, rightseq, # Full Join for item in rightseq: key = rightkey(item) - seen_keys.add(key) + seen(key) if key in d: for left_match in d[key]: yield (left_match, item) From b2eb929e4c88c6876a8510e5229842556a001245 Mon Sep 17 00:00:00 2001 From: Ryan Grout Date: Wed, 5 Dec 2018 23:06:56 -0600 Subject: [PATCH 62/74] Fix linting error. --- toolz/dicttoolz.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/toolz/dicttoolz.py b/toolz/dicttoolz.py index 84ba3a01..dbdd9eb7 100644 --- a/toolz/dicttoolz.py +++ b/toolz/dicttoolz.py @@ -7,6 +7,7 @@ 'valfilter', 'keyfilter', 'itemfilter', 'assoc', 'dissoc', 'assoc_in', 'update_in', 'get_in') + def _get_factory(f, kwargs): factory = kwargs.pop('factory', dict) if kwargs: @@ -288,6 +289,7 @@ def update_in(d, keys, func, default=None, factory=dict): inner[k] = func(default) return rv + def get_in(keys, coll, default=None, no_default=False): """ Returns coll[i0][i1]...[iX] where [i0, i1, ..., iX]==keys. From f058785eda01fabee1e6957bcdb09c365a33dbe1 Mon Sep 17 00:00:00 2001 From: Ryan Grout Date: Wed, 5 Dec 2018 23:59:33 -0600 Subject: [PATCH 63/74] Update tests for dissoc to use factory keyword argument. --- toolz/tests/test_dicttoolz.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/toolz/tests/test_dicttoolz.py b/toolz/tests/test_dicttoolz.py index 0226e49b..d7b78648 100644 --- a/toolz/tests/test_dicttoolz.py +++ b/toolz/tests/test_dicttoolz.py @@ -90,16 +90,16 @@ def test_assoc(self): def test_dissoc(self): D, kw = self.D, self.kw - assert dissoc(D({"a": 1}), "a") == D({}) - assert dissoc(D({"a": 1, "b": 2}), "a") == D({"b": 2}) - assert dissoc(D({"a": 1, "b": 2}), "b") == D({"a": 1}) - assert dissoc(D({"a": 1, "b": 2}), "a", "b") == D({}) - assert dissoc(D({"a": 1}), "a") == dissoc(dissoc(D({"a": 1}), "a"), "a") + assert dissoc(D({"a": 1}), "a", **kw) == D({}) + assert dissoc(D({"a": 1, "b": 2}), "a", **kw) == D({"b": 2}) + assert dissoc(D({"a": 1, "b": 2}), "b", **kw) == D({"a": 1}) + assert dissoc(D({"a": 1, "b": 2}), "a", "b", **kw) == D({}) + assert dissoc(D({"a": 1}), "a", **kw) == dissoc(dissoc(D({"a": 1}), "a", **kw), "a", **kw) # Verify immutability: d = D({'x': 1}) oldd = d - d2 = dissoc(d, 'x') + d2 = dissoc(d, 'x', **kw) assert d is oldd assert d2 is not oldd From 40ac54395cc22e212d95f292ca7005bc266bae09 Mon Sep 17 00:00:00 2001 From: dlovell Date: Sun, 23 Dec 2018 11:49:43 -0500 Subject: [PATCH 64/74] DOC/BUG: fix import so in memory split-apply-combine example runs --- doc/source/streaming-analytics.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/source/streaming-analytics.rst b/doc/source/streaming-analytics.rst index 9a8cdd06..99c002a4 100644 --- a/doc/source/streaming-analytics.rst +++ b/doc/source/streaming-analytics.rst @@ -81,8 +81,8 @@ groups. .. code:: - >>> from toolz import groupby, valmap, compose - >>> from toolz.curried import get, pluck + >>> from toolz import compose + >>> from toolz.curried import get, pluck, groupby, valmap >>> groupby(get(3), accounts) {'F': [(1, 'Alice', 100, 'F'), (5, 'Edith', 300, 'F')], From 41a382baf9b38c15e7fc26c745481dc42e0f352b Mon Sep 17 00:00:00 2001 From: Eric Yen Date: Thu, 10 Jan 2019 14:57:35 -0500 Subject: [PATCH 65/74] fix memory leak fix memory leak with objects contained in the exception / traceback. --- toolz/_signatures.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/toolz/_signatures.py b/toolz/_signatures.py index d2ad70fc..f34ebb79 100644 --- a/toolz/_signatures.py +++ b/toolz/_signatures.py @@ -642,7 +642,7 @@ def signature_or_spec(func): try: return inspect.signature(func) except (ValueError, TypeError) as e: - return e + return None else: # pragma: py3 no cover def num_pos_args(sigspec): @@ -664,7 +664,7 @@ def signature_or_spec(func): try: return inspect.getargspec(func) except TypeError as e: - return e + return None def expand_sig(sig): From cc49f898a5a81261f75b875b2bae0baa2e2409b9 Mon Sep 17 00:00:00 2001 From: Eric Yen Date: Mon, 21 Jan 2019 16:01:01 -0500 Subject: [PATCH 66/74] Remove unecessary definitions of exceptions --- toolz/_signatures.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/toolz/_signatures.py b/toolz/_signatures.py index f34ebb79..b9d87012 100644 --- a/toolz/_signatures.py +++ b/toolz/_signatures.py @@ -641,7 +641,7 @@ def get_exclude_keywords(num_pos_only, sigspec): def signature_or_spec(func): try: return inspect.signature(func) - except (ValueError, TypeError) as e: + except (ValueError, TypeError): return None else: # pragma: py3 no cover @@ -663,7 +663,7 @@ def get_exclude_keywords(num_pos_only, sigspec): def signature_or_spec(func): try: return inspect.getargspec(func) - except TypeError as e: + except TypeError: return None From f25bc8efaf4562e5831add5eab94fdd05eaf6213 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20=C5=A0uppa?= Date: Wed, 3 Apr 2019 01:43:10 +0200 Subject: [PATCH 67/74] Update streaming-analytics.rst Fix a link and a formatting in a note on `map` and `filter`. --- doc/source/streaming-analytics.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/streaming-analytics.rst b/doc/source/streaming-analytics.rst index 9a8cdd06..f37b3216 100644 --- a/doc/source/streaming-analytics.rst +++ b/doc/source/streaming-analytics.rst @@ -38,7 +38,7 @@ These functions correspond to the SQL commands ``SELECT`` and ``WHERE``. ... map(get([1, 2])), ... list) -*note: this uses the curried_ versions of ``map`` and ``filter``.* +Note: this uses the `curried`_ versions of ``map`` and ``filter``. Of course, these operations are also well supported with standard list/generator comprehension syntax. This syntax is more often used and From 7b6570a24c7e0213a91676e4e4139d5f738ff5f8 Mon Sep 17 00:00:00 2001 From: Mital Ashok Date: Thu, 31 Aug 2017 23:24:57 +0100 Subject: [PATCH 68/74] Added peekn --- doc/source/api.rst | 1 + toolz/curried/__init__.py | 1 + toolz/itertoolz.py | 22 ++++++++++++++++++++-- toolz/tests/test_itertoolz.py | 15 +++++++++++++-- 4 files changed, 35 insertions(+), 4 deletions(-) diff --git a/doc/source/api.rst b/doc/source/api.rst index f53ff19d..86ac4c0b 100644 --- a/doc/source/api.rst +++ b/doc/source/api.rst @@ -34,6 +34,7 @@ Itertoolz partition partition_all peek + peekn pluck random_sample reduceby diff --git a/toolz/curried/__init__.py b/toolz/curried/__init__.py index 43aeffd4..5a361952 100644 --- a/toolz/curried/__init__.py +++ b/toolz/curried/__init__.py @@ -80,6 +80,7 @@ partition = toolz.curry(toolz.partition) partition_all = toolz.curry(toolz.partition_all) partitionby = toolz.curry(toolz.partitionby) +peekn = toolz.curry(toolz.peekn) pluck = toolz.curry(toolz.pluck) random_sample = toolz.curry(toolz.random_sample) reduce = toolz.curry(toolz.reduce) diff --git a/toolz/itertoolz.py b/toolz/itertoolz.py index a25eea3c..54fe0c74 100644 --- a/toolz/itertoolz.py +++ b/toolz/itertoolz.py @@ -14,7 +14,7 @@ 'first', 'second', 'nth', 'last', 'get', 'concat', 'concatv', 'mapcat', 'cons', 'interpose', 'frequencies', 'reduceby', 'iterate', 'sliding_window', 'partition', 'partition_all', 'count', 'pluck', - 'join', 'tail', 'diff', 'topk', 'peek', 'random_sample') + 'join', 'tail', 'diff', 'topk', 'peek', 'peekn', 'random_sample') def remove(predicate, seq): @@ -942,7 +942,25 @@ def peek(seq): """ iterator = iter(seq) item = next(iterator) - return item, itertools.chain([item], iterator) + return item, itertools.chain((item,), iterator) + + +def peekn(n, seq): + """ Retrieve the next n elements of a sequence + + Returns a tuple of the first n elements and an iterable equivalent + to the original, still having the elements retrieved. + + >>> seq = [0, 1, 2, 3, 4] + >>> first_two, seq = peekn(2, seq) + >>> first_two + (0, 1) + >>> list(seq) + [0, 1, 2, 3, 4] + """ + iterator = iter(seq) + peeked = tuple(take(n, iterator)) + return peeked, itertools.chain(iter(peeked), iterator) def random_sample(prob, seq, random_state=None): diff --git a/toolz/tests/test_itertoolz.py b/toolz/tests/test_itertoolz.py index 93aa856d..bf06ab6f 100644 --- a/toolz/tests/test_itertoolz.py +++ b/toolz/tests/test_itertoolz.py @@ -13,7 +13,7 @@ reduceby, iterate, accumulate, sliding_window, count, partition, partition_all, take_nth, pluck, join, - diff, topk, peek, random_sample) + diff, topk, peek, peekn, random_sample) from toolz.compatibility import range, filter from operator import add, mul @@ -496,12 +496,23 @@ def test_topk_is_stable(): def test_peek(): alist = ["Alice", "Bob", "Carol"] element, blist = peek(alist) - element == alist[0] + assert element == alist[0] assert list(blist) == alist assert raises(StopIteration, lambda: peek([])) +def test_peekn(): + alist = ("Alice", "Bob", "Carol") + elements, blist = peekn(2, alist) + assert elements == alist[:2] + assert tuple(blist) == alist + + elements, blist = peekn(len(alist) * 4, alist) + assert elements == alist + assert tuple(blist) == alist + + def test_random_sample(): alist = list(range(100)) From 9e97f73fee5fc68bd03adcee3b520593ed5063d6 Mon Sep 17 00:00:00 2001 From: Sam Frances Date: Sun, 30 Jun 2019 14:22:33 +0100 Subject: [PATCH 69/74] Write pipeline function, with tests This involved some refactoring of the tests for compose(), so that we can programmatically ensure that compose() and pipeline() have equivalent tests. --- toolz/functoolz.py | 22 ++++++++ toolz/tests/test_functoolz.py | 99 +++++++++++++++++++++++++++++++---- 2 files changed, 111 insertions(+), 10 deletions(-) diff --git a/toolz/functoolz.py b/toolz/functoolz.py index 9091127d..e073b192 100644 --- a/toolz/functoolz.py +++ b/toolz/functoolz.py @@ -537,6 +537,7 @@ def compose(*funcs): '4' See Also: + pipeline pipe """ if not funcs: @@ -547,6 +548,27 @@ def compose(*funcs): return Compose(funcs) +def pipeline(*funcs): + """ Compose functions to operate in series. + + Returns a function that applies other functions in sequence. + + Functions are applied from left to right so that + ``mkpipe(f, g, h)(x, y)`` is the same as ``h(g(f(x, y)))``. + + If no arguments are provided, the identity function (f(x) = x) is returned. + + >>> inc = lambda i: i + 1 + >>> compose(inc, str)(3) + '4' + + See Also: + compose + pipe + """ + return compose(*reversed(funcs)) + + def pipe(data, *funcs): """ Pipe a value through a sequence of functions diff --git a/toolz/tests/test_functoolz.py b/toolz/tests/test_functoolz.py index 148c42ff..a43e9077 100644 --- a/toolz/tests/test_functoolz.py +++ b/toolz/tests/test_functoolz.py @@ -1,7 +1,9 @@ +import pytest import platform from toolz.functoolz import (thread_first, thread_last, memoize, curry, - compose, pipe, complement, do, juxt, flip, excepts, apply) + compose, pipeline, pipe, complement, do, juxt, + flip, excepts, apply) from operator import add, mul, itemgetter from toolz.utils import raises from functools import partial @@ -23,6 +25,10 @@ def double(x): return 2 * x +def add_then_multiply(a, b, c=10): + return (a + b) * c + + def test_apply(): assert apply(double, 5) == 10 assert tuple(map(apply, [double, inc, double], [10, 500, 8000])) == (20, 501, 16000) @@ -505,17 +511,67 @@ def _should_curry(self, args, kwargs, exc=None): """ -def test_compose(): - assert compose()(0) == 0 - assert compose(inc)(0) == 1 - assert compose(double, inc)(0) == 2 - assert compose(str, iseven, inc, double)(3) == "False" - assert compose(str, add)(1, 2) == '3' +def generate_compose_test_cases(): + """ + Generate test cases for parametrized tests of the compose function. + """ + return ( + ( + (), # arguments to compose() + (0,), {}, # positional and keyword args to the Composed object + 0 # expected result + ), + ( + (inc,), + (0,), {}, + 1 + ), + ( + (double, inc), + (0,), {}, + 2 + ), + ( + (str, iseven, inc, double), + (3,), {}, + "False" + ), + ( + (str, add), + (1, 2), {}, + '3' + ), + ( + (str, inc, add_then_multiply), + (1, 2), {"c": 3}, + '10' + ), + ) + + +def compose_test_ids(val): + """ + Generate readable test ids for test cases of the form generated by + generate_compose_test_cases(). + """ + try: + if type(val) in (dict, str): + raise TypeError + return "({})".format(", ".join(tuple(f.__name__ for f in val))) + except (AttributeError, TypeError): + return repr(val) - def f(a, b, c=10): - return (a + b) * c - assert compose(str, inc, f)(1, 2, c=3) == '10' +@pytest.mark.parametrize( + "compose_args,args,kwargs,expected", + generate_compose_test_cases(), + ids=compose_test_ids +) +def test_compose(compose_args, args, kwargs, expected): + assert compose(*compose_args)(*args, **kwargs) == expected + + +def test_compose_metadata(): # Define two functions with different names def f(a): @@ -536,6 +592,29 @@ def g(a): assert composed.__doc__ == 'A composition of functions' +def generate_pipeline_test_cases(): + """ + Generate test cases for parametrized tests of the compose function. + + These are based on, and equivalent to, those produced by + enerate_compose_test_cases(). + """ + return tuple( + (tuple(reversed(compose_args)), args, kwargs, expected) + for (compose_args, args, kwargs, expected) + in generate_compose_test_cases() + ) + + +@pytest.mark.parametrize( + "pipeline_args,args,kwargs,expected", + generate_pipeline_test_cases(), + ids=compose_test_ids +) +def test_pipeline(pipeline_args, args, kwargs, expected): + assert pipeline(*pipeline_args)(*args, **kwargs) == expected + + def test_pipe(): assert pipe(1, inc) == 2 assert pipe(1, inc, inc) == 3 From 9808e49b37f7efca27b006c2703f8e2f9ddd55a6 Mon Sep 17 00:00:00 2001 From: Sam Frances Date: Sun, 30 Jun 2019 14:48:55 +0100 Subject: [PATCH 70/74] Correct failing doctest --- toolz/functoolz.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/toolz/functoolz.py b/toolz/functoolz.py index e073b192..014c0f74 100644 --- a/toolz/functoolz.py +++ b/toolz/functoolz.py @@ -559,7 +559,7 @@ def pipeline(*funcs): If no arguments are provided, the identity function (f(x) = x) is returned. >>> inc = lambda i: i + 1 - >>> compose(inc, str)(3) + >>> pipeline(inc, str)(3) '4' See Also: From 3f4f6013b72b462c7a7adf154cbab372159575db Mon Sep 17 00:00:00 2001 From: Sam Frances Date: Sun, 30 Jun 2019 14:53:11 +0100 Subject: [PATCH 71/74] Convert pipeline() tests from pytest to nose This unfortunately removes ability of parametrized tests to fail independently, but is necessary so that CI tests will pass --- toolz/tests/test_functoolz.py | 42 +++++++++-------------------------- 1 file changed, 10 insertions(+), 32 deletions(-) diff --git a/toolz/tests/test_functoolz.py b/toolz/tests/test_functoolz.py index a43e9077..f62b7015 100644 --- a/toolz/tests/test_functoolz.py +++ b/toolz/tests/test_functoolz.py @@ -1,4 +1,3 @@ -import pytest import platform from toolz.functoolz import (thread_first, thread_last, memoize, curry, @@ -25,10 +24,6 @@ def double(x): return 2 * x -def add_then_multiply(a, b, c=10): - return (a + b) * c - - def test_apply(): assert apply(double, 5) == 10 assert tuple(map(apply, [double, inc, double], [10, 500, 8000])) == (20, 501, 16000) @@ -515,6 +510,10 @@ def generate_compose_test_cases(): """ Generate test cases for parametrized tests of the compose function. """ + + def add_then_multiply(a, b, c=10): + return (a + b) * c + return ( ( (), # arguments to compose() @@ -549,26 +548,9 @@ def generate_compose_test_cases(): ) -def compose_test_ids(val): - """ - Generate readable test ids for test cases of the form generated by - generate_compose_test_cases(). - """ - try: - if type(val) in (dict, str): - raise TypeError - return "({})".format(", ".join(tuple(f.__name__ for f in val))) - except (AttributeError, TypeError): - return repr(val) - - -@pytest.mark.parametrize( - "compose_args,args,kwargs,expected", - generate_compose_test_cases(), - ids=compose_test_ids -) -def test_compose(compose_args, args, kwargs, expected): - assert compose(*compose_args)(*args, **kwargs) == expected +def test_compose(): + for (compose_args, args, kw, expected) in generate_compose_test_cases(): + assert compose(*compose_args)(*args, **kw) == expected def test_compose_metadata(): @@ -606,13 +588,9 @@ def generate_pipeline_test_cases(): ) -@pytest.mark.parametrize( - "pipeline_args,args,kwargs,expected", - generate_pipeline_test_cases(), - ids=compose_test_ids -) -def test_pipeline(pipeline_args, args, kwargs, expected): - assert pipeline(*pipeline_args)(*args, **kwargs) == expected +def test_pipeline(): + for (pipeline_args, args, kw, expected) in generate_pipeline_test_cases(): + assert pipeline(*pipeline_args)(*args, **kw) == expected def test_pipe(): From 44983f75d794cf10d14676ba01b51d7a495e18f3 Mon Sep 17 00:00:00 2001 From: Sam Frances Date: Mon, 1 Jul 2019 17:07:12 +0100 Subject: [PATCH 72/74] Correct docstring for pipeline() --- toolz/functoolz.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/toolz/functoolz.py b/toolz/functoolz.py index 014c0f74..ee8874db 100644 --- a/toolz/functoolz.py +++ b/toolz/functoolz.py @@ -554,7 +554,7 @@ def pipeline(*funcs): Returns a function that applies other functions in sequence. Functions are applied from left to right so that - ``mkpipe(f, g, h)(x, y)`` is the same as ``h(g(f(x, y)))``. + ``pipeline(f, g, h)(x, y)`` is the same as ``h(g(f(x, y)))``. If no arguments are provided, the identity function (f(x) = x) is returned. From 1c07bc30dc3d2266e4b49765466b8eff26fadd10 Mon Sep 17 00:00:00 2001 From: Sam Frances Date: Mon, 8 Jul 2019 15:37:44 +0100 Subject: [PATCH 73/74] Rename pipline() to compose_left() --- toolz/functoolz.py | 12 ++++++------ toolz/tests/test_functoolz.py | 10 +++++----- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/toolz/functoolz.py b/toolz/functoolz.py index ee8874db..2ed180dc 100644 --- a/toolz/functoolz.py +++ b/toolz/functoolz.py @@ -10,8 +10,8 @@ __all__ = ('identity', 'apply', 'thread_first', 'thread_last', 'memoize', - 'compose', 'pipe', 'complement', 'juxt', 'do', 'curry', 'flip', - 'excepts') + 'compose', 'compose_left', 'pipe', 'complement', 'juxt', 'do', + 'curry', 'flip', 'excepts') def identity(x): @@ -537,7 +537,7 @@ def compose(*funcs): '4' See Also: - pipeline + compose_left pipe """ if not funcs: @@ -548,18 +548,18 @@ def compose(*funcs): return Compose(funcs) -def pipeline(*funcs): +def compose_left(*funcs): """ Compose functions to operate in series. Returns a function that applies other functions in sequence. Functions are applied from left to right so that - ``pipeline(f, g, h)(x, y)`` is the same as ``h(g(f(x, y)))``. + ``compose_left(f, g, h)(x, y)`` is the same as ``h(g(f(x, y)))``. If no arguments are provided, the identity function (f(x) = x) is returned. >>> inc = lambda i: i + 1 - >>> pipeline(inc, str)(3) + >>> compose_left(inc, str)(3) '4' See Also: diff --git a/toolz/tests/test_functoolz.py b/toolz/tests/test_functoolz.py index f62b7015..342327f5 100644 --- a/toolz/tests/test_functoolz.py +++ b/toolz/tests/test_functoolz.py @@ -1,7 +1,7 @@ import platform from toolz.functoolz import (thread_first, thread_last, memoize, curry, - compose, pipeline, pipe, complement, do, juxt, + compose, compose_left, pipe, complement, do, juxt, flip, excepts, apply) from operator import add, mul, itemgetter from toolz.utils import raises @@ -574,7 +574,7 @@ def g(a): assert composed.__doc__ == 'A composition of functions' -def generate_pipeline_test_cases(): +def generate_compose_left_test_cases(): """ Generate test cases for parametrized tests of the compose function. @@ -588,9 +588,9 @@ def generate_pipeline_test_cases(): ) -def test_pipeline(): - for (pipeline_args, args, kw, expected) in generate_pipeline_test_cases(): - assert pipeline(*pipeline_args)(*args, **kw) == expected +def test_compose_left(): + for (compose_left_args, args, kw, expected) in generate_compose_left_test_cases(): + assert compose_left(*compose_left_args)(*args, **kw) == expected def test_pipe(): From 1ef435f3f5347951bf8107bee20d9fa9aeaf06e9 Mon Sep 17 00:00:00 2001 From: Sam Frances Date: Mon, 8 Jul 2019 15:42:31 +0100 Subject: [PATCH 74/74] Add compose_left to toolz.curried --- toolz/curried/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/toolz/curried/__init__.py b/toolz/curried/__init__.py index fab6255f..80dadac6 100644 --- a/toolz/curried/__init__.py +++ b/toolz/curried/__init__.py @@ -30,6 +30,7 @@ comp, complement, compose, + compose_left, concat, concatv, count,