From b8cb4466fa974bcca53492d15bc0e2cbfb967990 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Sat, 28 Nov 2020 21:46:12 -0800 Subject: [PATCH 01/33] Complete the outer husk --- Modules/clinic/itertoolsmodule.c.h | 13 +++- Modules/itertoolsmodule.c | 106 ++++++++++++++++++++++++++++- 2 files changed, 116 insertions(+), 3 deletions(-) diff --git a/Modules/clinic/itertoolsmodule.c.h b/Modules/clinic/itertoolsmodule.c.h index c1192bbcb0d797d..80c39098dfe3cce 100644 --- a/Modules/clinic/itertoolsmodule.c.h +++ b/Modules/clinic/itertoolsmodule.c.h @@ -2,6 +2,17 @@ preserve [clinic start generated code]*/ +PyDoc_STRVAR(itertools_pairwise__doc__, +"pairwise($type, iterable, /)\n" +"--\n" +"\n" +"Return a pairwise object.\n" +"\n" +"\"s -> (s0,s1), (s1,s2), (s2, s3), ...\""); + +#define ITERTOOLS_PAIRWISE_METHODDEF \ + {"pairwise", (PyCFunction)itertools_pairwise, METH_O|METH_CLASS, itertools_pairwise__doc__}, + PyDoc_STRVAR(itertools_groupby__doc__, "groupby(iterable, key=None)\n" "--\n" @@ -627,4 +638,4 @@ itertools_count(PyTypeObject *type, PyObject *args, PyObject *kwargs) exit: return return_value; } -/*[clinic end generated code: output=d7f58dc477814b45 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=4b7af6880c7f7a4b input=a9049054013a1b77]*/ diff --git a/Modules/itertoolsmodule.c b/Modules/itertoolsmodule.c index ce8b4347ef220bb..cf435fd1eb4c4e0 100644 --- a/Modules/itertoolsmodule.c +++ b/Modules/itertoolsmodule.c @@ -27,8 +27,9 @@ class itertools.accumulate "accumulateobject *" "&accumulate_type" class itertools.compress "compressobject *" "&compress_type" class itertools.filterfalse "filterfalseobject *" "&filterfalse_type" class itertools.count "countobject *" "&count_type" +class itertools.pairwise "pairwiseobject *" "&pairwise_type" [clinic start generated code]*/ -/*[clinic end generated code: output=da39a3ee5e6b4b0d input=ea05c93c6d94726a]*/ +/*[clinic end generated code: output=da39a3ee5e6b4b0d input=6498ed21fbe1bf94]*/ static PyTypeObject groupby_type; static PyTypeObject _grouper_type; @@ -45,9 +46,109 @@ static PyTypeObject accumulate_type; static PyTypeObject compress_type; static PyTypeObject filterfalse_type; static PyTypeObject count_type; +static PyTypeObject pairwise_type; #include "clinic/itertoolsmodule.c.h" +/* pairwise object ***********************************************************/ + +typedef struct { + PyObject_HEAD + PyObject *it; + PyObject *old; +} pairwiseobject; + +/*[clinic input] +@classmethod +itertools.pairwise + iterable: object + / +Return a pairwise object. + +"s -> (s0,s1), (s1,s2), (s2, s3), ..." + +[clinic start generated code]*/ + +static PyObject * +itertools_pairwise(PyTypeObject *type, PyObject *iterable) +/*[clinic end generated code: output=aea36fd59aecff87 input=65db24d29d90acbe]*/ +{ + pairwiseobject *po; + + po = (pairwiseobject *)type->tp_alloc(type, 0); + if (po == NULL) { + return NULL; + } + po->it = PyObject_GetIter(iterable); + if (po->it == NULL) { + Py_DECREF(po); + return NULL; + } + po->old = NULL; + return (PyObject *)po; +} + +static void +pairwise_dealloc(pairwiseobject *po) +{ + PyObject_GC_UnTrack(po); + Py_XDECREF(po->it); + Py_XDECREF(po->old); + Py_TYPE(po)->tp_free(po); +} + +static int +pairwise_traverse(pairwiseobject *po, visitproc visit, void *arg) +{ + Py_VISIT(po->it); + Py_VISIT(po->old); + return 0; +} + +static PyTypeObject pairwise_type = { + PyVarObject_HEAD_INIT(&PyType_Type, 0) + "itertools.pairwise", /* tp_name */ + sizeof(pairwiseobject), /* tp_basicsize */ + 0, /* tp_itemsize */ + /* methods */ + (destructor)pairwise_dealloc, /* tp_dealloc */ + 0, /* tp_vectorcall_offset */ + 0, /* tp_getattr */ + 0, /* tp_setattr */ + 0, /* tp_as_async */ + 0, /* tp_repr */ + 0, /* tp_as_number */ + 0, /* tp_as_sequence */ + 0, /* tp_as_mapping */ + 0, /* tp_hash */ + 0, /* tp_call */ + 0, /* tp_str */ + PyObject_GenericGetAttr, /* tp_getattro */ + 0, /* tp_setattro */ + 0, /* tp_as_buffer */ + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | + Py_TPFLAGS_BASETYPE, /* tp_flags */ + itertools_pairwise__doc__, /* tp_doc */ + (traverseproc)pairwise_traverse, /* tp_traverse */ + 0, /* tp_clear */ + 0, /* tp_richcompare */ + 0, /* tp_weaklistoffset */ + PyObject_SelfIter, /* tp_iter */ + 0, /* (iternextfunc)enum_next, */ /* tp_iternext */ + 0, /* tp_methods */ + 0, /* tp_members */ + 0, /* tp_getset */ + 0, /* tp_base */ + 0, /* tp_dict */ + 0, /* tp_descr_get */ + 0, /* tp_descr_set */ + 0, /* tp_dictoffset */ + 0, /* tp_init */ + PyType_GenericAlloc, /* tp_alloc */ + itertools_pairwise, /* tp_new */ + PyObject_GC_Del, /* tp_free */ +}; + /* groupby object ************************************************************/ @@ -4701,7 +4802,8 @@ itertoolsmodule_exec(PyObject *m) &groupby_type, &_grouper_type, &tee_type, - &teedataobject_type + &teedataobject_type, + &pairwise_type }; Py_SET_TYPE(&teedataobject_type, &PyType_Type); From 8892f4038a28591f5fd572ffa3795bba7d9458bb Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Sat, 28 Nov 2020 22:30:06 -0800 Subject: [PATCH 02/33] First draft of __next__(). Always returns empty. --- Modules/itertoolsmodule.c | 28 +++++++++++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/Modules/itertoolsmodule.c b/Modules/itertoolsmodule.c index cf435fd1eb4c4e0..17289a5681e7d15 100644 --- a/Modules/itertoolsmodule.c +++ b/Modules/itertoolsmodule.c @@ -105,6 +105,32 @@ pairwise_traverse(pairwiseobject *po, visitproc visit, void *arg) return 0; } +static PyObject * +pairwise_next(pairwiseobject *po) +{ + PyObject *new, *result; + + if (po->it == NULL) { + return NULL; + } + if (po->old == NULL) { + po->old = PyIter_Next(po->it); + if (po->old == NULL) { + Py_CLEAR(po->it); + return NULL; + } + } + new = PyIter_Next(po->it); + if (new == NULL) { + Py_CLEAR(po->it); + Py_CLEAR(po->old); + return NULL; + } + result = PyTuple_Pack(2, po->old, new); + Py_SETREF(po->old, new); + return result; +} + static PyTypeObject pairwise_type = { PyVarObject_HEAD_INIT(&PyType_Type, 0) "itertools.pairwise", /* tp_name */ @@ -134,7 +160,7 @@ static PyTypeObject pairwise_type = { 0, /* tp_richcompare */ 0, /* tp_weaklistoffset */ PyObject_SelfIter, /* tp_iter */ - 0, /* (iternextfunc)enum_next, */ /* tp_iternext */ + (iternextfunc)pairwise_next, /* tp_iternext */ 0, /* tp_methods */ 0, /* tp_members */ 0, /* tp_getset */ From f86f5b1f09a15d0e793156b033096e66d098d9a3 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Sat, 28 Nov 2020 22:50:36 -0800 Subject: [PATCH 03/33] Fix signature of __new__ --- Modules/clinic/itertoolsmodule.c.h | 30 +++++++++++++++++++++++++----- Modules/itertoolsmodule.c | 10 +++++----- 2 files changed, 30 insertions(+), 10 deletions(-) diff --git a/Modules/clinic/itertoolsmodule.c.h b/Modules/clinic/itertoolsmodule.c.h index 80c39098dfe3cce..a382ac84e348ce4 100644 --- a/Modules/clinic/itertoolsmodule.c.h +++ b/Modules/clinic/itertoolsmodule.c.h @@ -2,16 +2,36 @@ preserve [clinic start generated code]*/ -PyDoc_STRVAR(itertools_pairwise__doc__, -"pairwise($type, iterable, /)\n" +PyDoc_STRVAR(pairwise_new__doc__, +"pairwise(iterable, /)\n" "--\n" "\n" "Return a pairwise object.\n" "\n" "\"s -> (s0,s1), (s1,s2), (s2, s3), ...\""); -#define ITERTOOLS_PAIRWISE_METHODDEF \ - {"pairwise", (PyCFunction)itertools_pairwise, METH_O|METH_CLASS, itertools_pairwise__doc__}, +static PyObject * +pairwise_new_impl(PyTypeObject *type, PyObject *iterable); + +static PyObject * +pairwise_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) +{ + PyObject *return_value = NULL; + PyObject *iterable; + + if ((type == &pairwise_type) && + !_PyArg_NoKeywords("pairwise", kwargs)) { + goto exit; + } + if (!_PyArg_CheckPositional("pairwise", PyTuple_GET_SIZE(args), 1, 1)) { + goto exit; + } + iterable = PyTuple_GET_ITEM(args, 0); + return_value = pairwise_new_impl(type, iterable); + +exit: + return return_value; +} PyDoc_STRVAR(itertools_groupby__doc__, "groupby(iterable, key=None)\n" @@ -638,4 +658,4 @@ itertools_count(PyTypeObject *type, PyObject *args, PyObject *kwargs) exit: return return_value; } -/*[clinic end generated code: output=4b7af6880c7f7a4b input=a9049054013a1b77]*/ +/*[clinic end generated code: output=540324a5db73ac98 input=a9049054013a1b77]*/ diff --git a/Modules/itertoolsmodule.c b/Modules/itertoolsmodule.c index 17289a5681e7d15..5b37341161e5f4c 100644 --- a/Modules/itertoolsmodule.c +++ b/Modules/itertoolsmodule.c @@ -60,7 +60,7 @@ typedef struct { /*[clinic input] @classmethod -itertools.pairwise +itertools.pairwise.__new__ as pairwise_new iterable: object / Return a pairwise object. @@ -70,8 +70,8 @@ Return a pairwise object. [clinic start generated code]*/ static PyObject * -itertools_pairwise(PyTypeObject *type, PyObject *iterable) -/*[clinic end generated code: output=aea36fd59aecff87 input=65db24d29d90acbe]*/ +pairwise_new_impl(PyTypeObject *type, PyObject *iterable) +/*[clinic end generated code: output=9f0267062d384456 input=a1ad925c82e0e901]*/ { pairwiseobject *po; @@ -154,7 +154,7 @@ static PyTypeObject pairwise_type = { 0, /* tp_as_buffer */ Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_BASETYPE, /* tp_flags */ - itertools_pairwise__doc__, /* tp_doc */ + pairwise_new__doc__, /* tp_doc */ (traverseproc)pairwise_traverse, /* tp_traverse */ 0, /* tp_clear */ 0, /* tp_richcompare */ @@ -171,7 +171,7 @@ static PyTypeObject pairwise_type = { 0, /* tp_dictoffset */ 0, /* tp_init */ PyType_GenericAlloc, /* tp_alloc */ - itertools_pairwise, /* tp_new */ + pairwise_new, /* tp_new */ PyObject_GC_Del, /* tp_free */ }; From 69884cb2e08fbe22a86a4d0448067711de601356 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Sat, 28 Nov 2020 22:53:08 -0800 Subject: [PATCH 04/33] Add blurb --- .../NEWS.d/next/Library/2020-11-28-22-52-57.bpo-38200.DuWGlW.rst | 1 + 1 file changed, 1 insertion(+) create mode 100644 Misc/NEWS.d/next/Library/2020-11-28-22-52-57.bpo-38200.DuWGlW.rst diff --git a/Misc/NEWS.d/next/Library/2020-11-28-22-52-57.bpo-38200.DuWGlW.rst b/Misc/NEWS.d/next/Library/2020-11-28-22-52-57.bpo-38200.DuWGlW.rst new file mode 100644 index 000000000000000..b4bc5551b2532ec --- /dev/null +++ b/Misc/NEWS.d/next/Library/2020-11-28-22-52-57.bpo-38200.DuWGlW.rst @@ -0,0 +1 @@ +Added itertools.pairwise() From 6fbf203c3d1489c5c89e03c21970d39c036094b6 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Sat, 28 Nov 2020 22:58:33 -0800 Subject: [PATCH 05/33] Remove old recipe --- Doc/library/itertools.rst | 6 ------ 1 file changed, 6 deletions(-) diff --git a/Doc/library/itertools.rst b/Doc/library/itertools.rst index 107bc515a677856..0818e2729aa8841 100644 --- a/Doc/library/itertools.rst +++ b/Doc/library/itertools.rst @@ -782,12 +782,6 @@ which incur interpreter overhead. return starmap(func, repeat(args)) return starmap(func, repeat(args, times)) - def pairwise(iterable): - "s -> (s0,s1), (s1,s2), (s2, s3), ..." - a, b = tee(iterable) - next(b, None) - return zip(a, b) - def grouper(iterable, n, fillvalue=None): "Collect data into fixed-length chunks or blocks" # grouper('ABCDEFG', 3, 'x') --> ABC DEF Gxx" From c7d5ac7e36287fc663a0344781986cbfc9b0aae2 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Sun, 29 Nov 2020 01:19:29 -0800 Subject: [PATCH 06/33] Remove old pairwise() recipe tests --- Lib/test/test_itertools.py | 9 --------- 1 file changed, 9 deletions(-) diff --git a/Lib/test/test_itertools.py b/Lib/test/test_itertools.py index eaa6197bec395c6..775ffa9e6e5c8ad 100644 --- a/Lib/test/test_itertools.py +++ b/Lib/test/test_itertools.py @@ -2451,15 +2451,6 @@ def test_permutations_sizeof(self): >>> take(5, map(int, repeatfunc(random.random))) [0, 0, 0, 0, 0] ->>> list(pairwise('abcd')) -[('a', 'b'), ('b', 'c'), ('c', 'd')] - ->>> list(pairwise([])) -[] - ->>> list(pairwise('a')) -[] - >>> list(islice(padnone('abc'), 0, 6)) ['a', 'b', 'c', None, None, None] From 1d5472c52cb1f6dd4c0c48859e7224af193e956e Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Sun, 29 Nov 2020 01:20:10 -0800 Subject: [PATCH 07/33] Remove old pairwise() recipe tests --- Lib/test/test_itertools.py | 9 --------- 1 file changed, 9 deletions(-) diff --git a/Lib/test/test_itertools.py b/Lib/test/test_itertools.py index 775ffa9e6e5c8ad..3caf316e81e3273 100644 --- a/Lib/test/test_itertools.py +++ b/Lib/test/test_itertools.py @@ -2312,15 +2312,6 @@ def test_permutations_sizeof(self): ... else: ... return starmap(func, repeat(args, times)) ->>> def pairwise(iterable): -... "s -> (s0,s1), (s1,s2), (s2, s3), ..." -... a, b = tee(iterable) -... try: -... next(b) -... except StopIteration: -... pass -... return zip(a, b) - >>> def grouper(n, iterable, fillvalue=None): ... "grouper(3, 'ABCDEFG', 'x') --> ABC DEF Gxx" ... args = [iter(iterable)] * n From 97faaa5f713d6387e224c4e1a3718004140074a5 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Sun, 29 Nov 2020 02:22:20 -0800 Subject: [PATCH 08/33] Add tests --- Lib/test/test_itertools.py | 34 ++++++++++++++++++++++++++++++++++ Modules/itertoolsmodule.c | 2 +- 2 files changed, 35 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_itertools.py b/Lib/test/test_itertools.py index 3caf316e81e3273..73fd9410b5fbb3f 100644 --- a/Lib/test/test_itertools.py +++ b/Lib/test/test_itertools.py @@ -1024,6 +1024,25 @@ def run(r1, r2): self.assertEqual(next(it), (1, 2)) self.assertRaises(RuntimeError, next, it) + def test_pairwise(self): + self.assertEqual(list(pairwise('')), []) + self.assertEqual(list(pairwise('a')), []) + self.assertEqual(list(pairwise('ab')), + [('a', 'b')]), + self.assertEqual(list(pairwise('abcde')), + [('a', 'b'), ('b', 'c'), ('c', 'd'), ('d', 'e')]) + self.assertEqual(list(pairwise(range(10_000))), + list(zip(range(10_000), range(1, 10_000)))) + + with self.assertRaises(TypeError): + pairwise() # too few arguments + with self.assertRaises(TypeError): + pairwise('abc', 10) # too many arguments + with self.assertRaises(TypeError): + pairwise(iterable='abc') # keyword arguments + with self.assertRaises(TypeError): + pairwise(None) # non-iterable argument + def test_product(self): for args, result in [ ([], [()]), # zero iterables @@ -1787,6 +1806,10 @@ def test_islice(self): a = [] self.makecycle(islice([a]*2, None), a) + def test_pairwise(self): + a = [] + self.makecycle(pairwise([a]*5), a) + def test_permutations(self): a = [] self.makecycle(permutations([1,2,a,3], 3), a) @@ -1995,6 +2018,17 @@ def test_islice(self): self.assertRaises(TypeError, islice, N(s), 10) self.assertRaises(ZeroDivisionError, list, islice(E(s), 10)) + def test_pairwise(self): + for s in ("123", "", range(1000), ('do', 1.2), range(2000,2200,5)): + for g in (G, I, Ig, S, L, R): + seq = list(g(s)) + expected = list(zip(seq, seq[1:])) + actual = list(pairwise(g(s))) + self.assertEqual(actual, expected) + self.assertRaises(TypeError, pairwise, X(s)) + self.assertRaises(TypeError, pairwise, N(s)) + self.assertRaises(ZeroDivisionError, list, pairwise(E(s))) + def test_starmap(self): for s in (range(10), range(0), range(100), (7,11), range(20,50,5)): for g in (G, I, Ig, S, L, R): diff --git a/Modules/itertoolsmodule.c b/Modules/itertoolsmodule.c index 5b37341161e5f4c..956b1c7dcb429fb 100644 --- a/Modules/itertoolsmodule.c +++ b/Modules/itertoolsmodule.c @@ -79,12 +79,12 @@ pairwise_new_impl(PyTypeObject *type, PyObject *iterable) if (po == NULL) { return NULL; } + po->old = NULL; po->it = PyObject_GetIter(iterable); if (po->it == NULL) { Py_DECREF(po); return NULL; } - po->old = NULL; return (PyObject *)po; } From ca6ab39c6bb239826d49d0eab6ba5839c3624b8b Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Sun, 29 Nov 2020 02:47:29 -0800 Subject: [PATCH 09/33] Tighten-up the code a bit --- Modules/itertoolsmodule.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/Modules/itertoolsmodule.c b/Modules/itertoolsmodule.c index 956b1c7dcb429fb..25e8cbf872a73bb 100644 --- a/Modules/itertoolsmodule.c +++ b/Modules/itertoolsmodule.c @@ -108,19 +108,20 @@ pairwise_traverse(pairwiseobject *po, visitproc visit, void *arg) static PyObject * pairwise_next(pairwiseobject *po) { + PyObject *it = po->it; PyObject *new, *result; - if (po->it == NULL) { + if (it == NULL) { return NULL; } if (po->old == NULL) { - po->old = PyIter_Next(po->it); + po->old = (*Py_TYPE(it)->tp_iternext)(it); if (po->old == NULL) { Py_CLEAR(po->it); return NULL; } } - new = PyIter_Next(po->it); + new = (*Py_TYPE(it)->tp_iternext)(it); if (new == NULL) { Py_CLEAR(po->it); Py_CLEAR(po->old); From ce4d46dc282c71003bbc4e7963b1789151dbda63 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Sun, 29 Nov 2020 03:19:56 -0800 Subject: [PATCH 10/33] Clean-up __new__ by putting the iterator test first --- Modules/itertoolsmodule.c | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/Modules/itertoolsmodule.c b/Modules/itertoolsmodule.c index 25e8cbf872a73bb..998315c794b97a1 100644 --- a/Modules/itertoolsmodule.c +++ b/Modules/itertoolsmodule.c @@ -73,18 +73,21 @@ static PyObject * pairwise_new_impl(PyTypeObject *type, PyObject *iterable) /*[clinic end generated code: output=9f0267062d384456 input=a1ad925c82e0e901]*/ { + PyObject *it; pairwiseobject *po; + it = PyObject_GetIter(iterable); + if (it == NULL) { + return NULL; + } + po = (pairwiseobject *)type->tp_alloc(type, 0); if (po == NULL) { + Py_DECREF(it); return NULL; } + po->it = it; po->old = NULL; - po->it = PyObject_GetIter(iterable); - if (po->it == NULL) { - Py_DECREF(po); - return NULL; - } return (PyObject *)po; } From da4ecace64dfa4eeebfdc71e684f58fedf12f9f2 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Sun, 29 Nov 2020 03:29:04 -0800 Subject: [PATCH 11/33] Don't lookup the old variable twice --- Modules/itertoolsmodule.c | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/Modules/itertoolsmodule.c b/Modules/itertoolsmodule.c index 998315c794b97a1..8c6e95c2e6db3be 100644 --- a/Modules/itertoolsmodule.c +++ b/Modules/itertoolsmodule.c @@ -112,14 +112,15 @@ static PyObject * pairwise_next(pairwiseobject *po) { PyObject *it = po->it; + PyObject *old = po->old; PyObject *new, *result; if (it == NULL) { return NULL; } - if (po->old == NULL) { - po->old = (*Py_TYPE(it)->tp_iternext)(it); - if (po->old == NULL) { + if (old == NULL) { + po->old = old = (*Py_TYPE(it)->tp_iternext)(it); + if (old == NULL) { Py_CLEAR(po->it); return NULL; } @@ -130,7 +131,7 @@ pairwise_next(pairwiseobject *po) Py_CLEAR(po->old); return NULL; } - result = PyTuple_Pack(2, po->old, new); + result = PyTuple_Pack(2, old, new); Py_SETREF(po->old, new); return result; } From 649353c055d9c2b5d565b3f32a0ad29c6d9ffa66 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Sun, 29 Nov 2020 03:36:31 -0800 Subject: [PATCH 12/33] Minor change in ordering --- Modules/itertoolsmodule.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Modules/itertoolsmodule.c b/Modules/itertoolsmodule.c index 8c6e95c2e6db3be..3e65b396bd35aa6 100644 --- a/Modules/itertoolsmodule.c +++ b/Modules/itertoolsmodule.c @@ -1,4 +1,5 @@ + #define PY_SSIZE_T_CLEAN #include "Python.h" #include "pycore_long.h" // _PyLong_GetZero() @@ -4827,6 +4828,7 @@ itertoolsmodule_exec(PyObject *m) &filterfalse_type, &count_type, &ziplongest_type, + &pairwise_type &permutations_type, &product_type, &repeat_type, @@ -4834,7 +4836,6 @@ itertoolsmodule_exec(PyObject *m) &_grouper_type, &tee_type, &teedataobject_type, - &pairwise_type }; Py_SET_TYPE(&teedataobject_type, &PyType_Type); From ec37fcb42120c665fa45a9ff6dbe7476264adbb3 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Sun, 29 Nov 2020 03:37:33 -0800 Subject: [PATCH 13/33] Fix missing comma --- Modules/itertoolsmodule.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Modules/itertoolsmodule.c b/Modules/itertoolsmodule.c index 3e65b396bd35aa6..d0e1fb3af2868e9 100644 --- a/Modules/itertoolsmodule.c +++ b/Modules/itertoolsmodule.c @@ -4828,7 +4828,7 @@ itertoolsmodule_exec(PyObject *m) &filterfalse_type, &count_type, &ziplongest_type, - &pairwise_type + &pairwise_type, &permutations_type, &product_type, &repeat_type, From bf6f0ef419aa9f13c90a824d407a0f7fc14cb7d5 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Sun, 29 Nov 2020 03:38:38 -0800 Subject: [PATCH 14/33] Don't unnecessarily change a line in the original code --- Modules/itertoolsmodule.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Modules/itertoolsmodule.c b/Modules/itertoolsmodule.c index d0e1fb3af2868e9..9a905db3809f4e2 100644 --- a/Modules/itertoolsmodule.c +++ b/Modules/itertoolsmodule.c @@ -4835,7 +4835,7 @@ itertoolsmodule_exec(PyObject *m) &groupby_type, &_grouper_type, &tee_type, - &teedataobject_type, + &teedataobject_type }; Py_SET_TYPE(&teedataobject_type, &PyType_Type); From d68b408ca696c3c4c5aa62dd702fb076aa6c496f Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Sun, 29 Nov 2020 11:00:33 -0800 Subject: [PATCH 15/33] Add todo comment --- Modules/itertoolsmodule.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Modules/itertoolsmodule.c b/Modules/itertoolsmodule.c index 9a905db3809f4e2..501b97dbc46c7e5 100644 --- a/Modules/itertoolsmodule.c +++ b/Modules/itertoolsmodule.c @@ -81,7 +81,6 @@ pairwise_new_impl(PyTypeObject *type, PyObject *iterable) if (it == NULL) { return NULL; } - po = (pairwiseobject *)type->tp_alloc(type, 0); if (po == NULL) { Py_DECREF(it); @@ -132,6 +131,7 @@ pairwise_next(pairwiseobject *po) Py_CLEAR(po->old); return NULL; } + /* Future optimization: Reuse the result tuple as we do in enumerate() */ result = PyTuple_Pack(2, old, new); Py_SETREF(po->old, new); return result; From e194bc9fe44a0fc079b8627065403bfd4de2bc2f Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Sat, 28 Nov 2020 21:46:12 -0800 Subject: [PATCH 16/33] Complete the outer husk --- Modules/clinic/itertoolsmodule.c.h | 13 +++- Modules/itertoolsmodule.c | 106 ++++++++++++++++++++++++++++- 2 files changed, 116 insertions(+), 3 deletions(-) diff --git a/Modules/clinic/itertoolsmodule.c.h b/Modules/clinic/itertoolsmodule.c.h index c1192bbcb0d797d..80c39098dfe3cce 100644 --- a/Modules/clinic/itertoolsmodule.c.h +++ b/Modules/clinic/itertoolsmodule.c.h @@ -2,6 +2,17 @@ preserve [clinic start generated code]*/ +PyDoc_STRVAR(itertools_pairwise__doc__, +"pairwise($type, iterable, /)\n" +"--\n" +"\n" +"Return a pairwise object.\n" +"\n" +"\"s -> (s0,s1), (s1,s2), (s2, s3), ...\""); + +#define ITERTOOLS_PAIRWISE_METHODDEF \ + {"pairwise", (PyCFunction)itertools_pairwise, METH_O|METH_CLASS, itertools_pairwise__doc__}, + PyDoc_STRVAR(itertools_groupby__doc__, "groupby(iterable, key=None)\n" "--\n" @@ -627,4 +638,4 @@ itertools_count(PyTypeObject *type, PyObject *args, PyObject *kwargs) exit: return return_value; } -/*[clinic end generated code: output=d7f58dc477814b45 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=4b7af6880c7f7a4b input=a9049054013a1b77]*/ diff --git a/Modules/itertoolsmodule.c b/Modules/itertoolsmodule.c index ce8b4347ef220bb..cf435fd1eb4c4e0 100644 --- a/Modules/itertoolsmodule.c +++ b/Modules/itertoolsmodule.c @@ -27,8 +27,9 @@ class itertools.accumulate "accumulateobject *" "&accumulate_type" class itertools.compress "compressobject *" "&compress_type" class itertools.filterfalse "filterfalseobject *" "&filterfalse_type" class itertools.count "countobject *" "&count_type" +class itertools.pairwise "pairwiseobject *" "&pairwise_type" [clinic start generated code]*/ -/*[clinic end generated code: output=da39a3ee5e6b4b0d input=ea05c93c6d94726a]*/ +/*[clinic end generated code: output=da39a3ee5e6b4b0d input=6498ed21fbe1bf94]*/ static PyTypeObject groupby_type; static PyTypeObject _grouper_type; @@ -45,9 +46,109 @@ static PyTypeObject accumulate_type; static PyTypeObject compress_type; static PyTypeObject filterfalse_type; static PyTypeObject count_type; +static PyTypeObject pairwise_type; #include "clinic/itertoolsmodule.c.h" +/* pairwise object ***********************************************************/ + +typedef struct { + PyObject_HEAD + PyObject *it; + PyObject *old; +} pairwiseobject; + +/*[clinic input] +@classmethod +itertools.pairwise + iterable: object + / +Return a pairwise object. + +"s -> (s0,s1), (s1,s2), (s2, s3), ..." + +[clinic start generated code]*/ + +static PyObject * +itertools_pairwise(PyTypeObject *type, PyObject *iterable) +/*[clinic end generated code: output=aea36fd59aecff87 input=65db24d29d90acbe]*/ +{ + pairwiseobject *po; + + po = (pairwiseobject *)type->tp_alloc(type, 0); + if (po == NULL) { + return NULL; + } + po->it = PyObject_GetIter(iterable); + if (po->it == NULL) { + Py_DECREF(po); + return NULL; + } + po->old = NULL; + return (PyObject *)po; +} + +static void +pairwise_dealloc(pairwiseobject *po) +{ + PyObject_GC_UnTrack(po); + Py_XDECREF(po->it); + Py_XDECREF(po->old); + Py_TYPE(po)->tp_free(po); +} + +static int +pairwise_traverse(pairwiseobject *po, visitproc visit, void *arg) +{ + Py_VISIT(po->it); + Py_VISIT(po->old); + return 0; +} + +static PyTypeObject pairwise_type = { + PyVarObject_HEAD_INIT(&PyType_Type, 0) + "itertools.pairwise", /* tp_name */ + sizeof(pairwiseobject), /* tp_basicsize */ + 0, /* tp_itemsize */ + /* methods */ + (destructor)pairwise_dealloc, /* tp_dealloc */ + 0, /* tp_vectorcall_offset */ + 0, /* tp_getattr */ + 0, /* tp_setattr */ + 0, /* tp_as_async */ + 0, /* tp_repr */ + 0, /* tp_as_number */ + 0, /* tp_as_sequence */ + 0, /* tp_as_mapping */ + 0, /* tp_hash */ + 0, /* tp_call */ + 0, /* tp_str */ + PyObject_GenericGetAttr, /* tp_getattro */ + 0, /* tp_setattro */ + 0, /* tp_as_buffer */ + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | + Py_TPFLAGS_BASETYPE, /* tp_flags */ + itertools_pairwise__doc__, /* tp_doc */ + (traverseproc)pairwise_traverse, /* tp_traverse */ + 0, /* tp_clear */ + 0, /* tp_richcompare */ + 0, /* tp_weaklistoffset */ + PyObject_SelfIter, /* tp_iter */ + 0, /* (iternextfunc)enum_next, */ /* tp_iternext */ + 0, /* tp_methods */ + 0, /* tp_members */ + 0, /* tp_getset */ + 0, /* tp_base */ + 0, /* tp_dict */ + 0, /* tp_descr_get */ + 0, /* tp_descr_set */ + 0, /* tp_dictoffset */ + 0, /* tp_init */ + PyType_GenericAlloc, /* tp_alloc */ + itertools_pairwise, /* tp_new */ + PyObject_GC_Del, /* tp_free */ +}; + /* groupby object ************************************************************/ @@ -4701,7 +4802,8 @@ itertoolsmodule_exec(PyObject *m) &groupby_type, &_grouper_type, &tee_type, - &teedataobject_type + &teedataobject_type, + &pairwise_type }; Py_SET_TYPE(&teedataobject_type, &PyType_Type); From 0536759bb7e93d5d60f9f9043ff61ad546be09b3 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Sat, 28 Nov 2020 22:30:06 -0800 Subject: [PATCH 17/33] First draft of __next__(). Always returns empty. --- Modules/itertoolsmodule.c | 28 +++++++++++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/Modules/itertoolsmodule.c b/Modules/itertoolsmodule.c index cf435fd1eb4c4e0..17289a5681e7d15 100644 --- a/Modules/itertoolsmodule.c +++ b/Modules/itertoolsmodule.c @@ -105,6 +105,32 @@ pairwise_traverse(pairwiseobject *po, visitproc visit, void *arg) return 0; } +static PyObject * +pairwise_next(pairwiseobject *po) +{ + PyObject *new, *result; + + if (po->it == NULL) { + return NULL; + } + if (po->old == NULL) { + po->old = PyIter_Next(po->it); + if (po->old == NULL) { + Py_CLEAR(po->it); + return NULL; + } + } + new = PyIter_Next(po->it); + if (new == NULL) { + Py_CLEAR(po->it); + Py_CLEAR(po->old); + return NULL; + } + result = PyTuple_Pack(2, po->old, new); + Py_SETREF(po->old, new); + return result; +} + static PyTypeObject pairwise_type = { PyVarObject_HEAD_INIT(&PyType_Type, 0) "itertools.pairwise", /* tp_name */ @@ -134,7 +160,7 @@ static PyTypeObject pairwise_type = { 0, /* tp_richcompare */ 0, /* tp_weaklistoffset */ PyObject_SelfIter, /* tp_iter */ - 0, /* (iternextfunc)enum_next, */ /* tp_iternext */ + (iternextfunc)pairwise_next, /* tp_iternext */ 0, /* tp_methods */ 0, /* tp_members */ 0, /* tp_getset */ From 8440b5d6e0bed5004d25bb9c380547c05bb98303 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Sat, 28 Nov 2020 22:50:36 -0800 Subject: [PATCH 18/33] Fix signature of __new__ --- Modules/clinic/itertoolsmodule.c.h | 30 +++++++++++++++++++++++++----- Modules/itertoolsmodule.c | 10 +++++----- 2 files changed, 30 insertions(+), 10 deletions(-) diff --git a/Modules/clinic/itertoolsmodule.c.h b/Modules/clinic/itertoolsmodule.c.h index 80c39098dfe3cce..a382ac84e348ce4 100644 --- a/Modules/clinic/itertoolsmodule.c.h +++ b/Modules/clinic/itertoolsmodule.c.h @@ -2,16 +2,36 @@ preserve [clinic start generated code]*/ -PyDoc_STRVAR(itertools_pairwise__doc__, -"pairwise($type, iterable, /)\n" +PyDoc_STRVAR(pairwise_new__doc__, +"pairwise(iterable, /)\n" "--\n" "\n" "Return a pairwise object.\n" "\n" "\"s -> (s0,s1), (s1,s2), (s2, s3), ...\""); -#define ITERTOOLS_PAIRWISE_METHODDEF \ - {"pairwise", (PyCFunction)itertools_pairwise, METH_O|METH_CLASS, itertools_pairwise__doc__}, +static PyObject * +pairwise_new_impl(PyTypeObject *type, PyObject *iterable); + +static PyObject * +pairwise_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) +{ + PyObject *return_value = NULL; + PyObject *iterable; + + if ((type == &pairwise_type) && + !_PyArg_NoKeywords("pairwise", kwargs)) { + goto exit; + } + if (!_PyArg_CheckPositional("pairwise", PyTuple_GET_SIZE(args), 1, 1)) { + goto exit; + } + iterable = PyTuple_GET_ITEM(args, 0); + return_value = pairwise_new_impl(type, iterable); + +exit: + return return_value; +} PyDoc_STRVAR(itertools_groupby__doc__, "groupby(iterable, key=None)\n" @@ -638,4 +658,4 @@ itertools_count(PyTypeObject *type, PyObject *args, PyObject *kwargs) exit: return return_value; } -/*[clinic end generated code: output=4b7af6880c7f7a4b input=a9049054013a1b77]*/ +/*[clinic end generated code: output=540324a5db73ac98 input=a9049054013a1b77]*/ diff --git a/Modules/itertoolsmodule.c b/Modules/itertoolsmodule.c index 17289a5681e7d15..5b37341161e5f4c 100644 --- a/Modules/itertoolsmodule.c +++ b/Modules/itertoolsmodule.c @@ -60,7 +60,7 @@ typedef struct { /*[clinic input] @classmethod -itertools.pairwise +itertools.pairwise.__new__ as pairwise_new iterable: object / Return a pairwise object. @@ -70,8 +70,8 @@ Return a pairwise object. [clinic start generated code]*/ static PyObject * -itertools_pairwise(PyTypeObject *type, PyObject *iterable) -/*[clinic end generated code: output=aea36fd59aecff87 input=65db24d29d90acbe]*/ +pairwise_new_impl(PyTypeObject *type, PyObject *iterable) +/*[clinic end generated code: output=9f0267062d384456 input=a1ad925c82e0e901]*/ { pairwiseobject *po; @@ -154,7 +154,7 @@ static PyTypeObject pairwise_type = { 0, /* tp_as_buffer */ Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_BASETYPE, /* tp_flags */ - itertools_pairwise__doc__, /* tp_doc */ + pairwise_new__doc__, /* tp_doc */ (traverseproc)pairwise_traverse, /* tp_traverse */ 0, /* tp_clear */ 0, /* tp_richcompare */ @@ -171,7 +171,7 @@ static PyTypeObject pairwise_type = { 0, /* tp_dictoffset */ 0, /* tp_init */ PyType_GenericAlloc, /* tp_alloc */ - itertools_pairwise, /* tp_new */ + pairwise_new, /* tp_new */ PyObject_GC_Del, /* tp_free */ }; From 5b7b13d99cdd514b2ca006e556ca5eaa447917d1 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Sat, 28 Nov 2020 22:53:08 -0800 Subject: [PATCH 19/33] Add blurb --- .../NEWS.d/next/Library/2020-11-28-22-52-57.bpo-38200.DuWGlW.rst | 1 + 1 file changed, 1 insertion(+) create mode 100644 Misc/NEWS.d/next/Library/2020-11-28-22-52-57.bpo-38200.DuWGlW.rst diff --git a/Misc/NEWS.d/next/Library/2020-11-28-22-52-57.bpo-38200.DuWGlW.rst b/Misc/NEWS.d/next/Library/2020-11-28-22-52-57.bpo-38200.DuWGlW.rst new file mode 100644 index 000000000000000..b4bc5551b2532ec --- /dev/null +++ b/Misc/NEWS.d/next/Library/2020-11-28-22-52-57.bpo-38200.DuWGlW.rst @@ -0,0 +1 @@ +Added itertools.pairwise() From ffc7e422a53d26b9aaf22d3c304f93d2347004ab Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Sat, 28 Nov 2020 22:58:33 -0800 Subject: [PATCH 20/33] Remove old recipe --- Doc/library/itertools.rst | 6 ------ 1 file changed, 6 deletions(-) diff --git a/Doc/library/itertools.rst b/Doc/library/itertools.rst index 3de66c934928157..392a370b880f3a3 100644 --- a/Doc/library/itertools.rst +++ b/Doc/library/itertools.rst @@ -782,12 +782,6 @@ which incur interpreter overhead. return starmap(func, repeat(args)) return starmap(func, repeat(args, times)) - def pairwise(iterable): - "s -> (s0,s1), (s1,s2), (s2, s3), ..." - a, b = tee(iterable) - next(b, None) - return zip(a, b) - def grouper(iterable, n, fillvalue=None): "Collect data into fixed-length chunks or blocks" # grouper('ABCDEFG', 3, 'x') --> ABC DEF Gxx" From cd659c350dc8ccea3151b36f5937041b97161411 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Sun, 29 Nov 2020 01:19:29 -0800 Subject: [PATCH 21/33] Remove old pairwise() recipe tests --- Lib/test/test_itertools.py | 9 --------- 1 file changed, 9 deletions(-) diff --git a/Lib/test/test_itertools.py b/Lib/test/test_itertools.py index 702cf0820316b19..d84fd85357ac89d 100644 --- a/Lib/test/test_itertools.py +++ b/Lib/test/test_itertools.py @@ -2451,15 +2451,6 @@ def test_permutations_sizeof(self): >>> take(5, map(int, repeatfunc(random.random))) [0, 0, 0, 0, 0] ->>> list(pairwise('abcd')) -[('a', 'b'), ('b', 'c'), ('c', 'd')] - ->>> list(pairwise([])) -[] - ->>> list(pairwise('a')) -[] - >>> list(islice(pad_none('abc'), 0, 6)) ['a', 'b', 'c', None, None, None] From 768f7a784789e5e2267da776dd1cc212af5b5fd2 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Sun, 29 Nov 2020 01:20:10 -0800 Subject: [PATCH 22/33] Remove old pairwise() recipe tests --- Lib/test/test_itertools.py | 9 --------- 1 file changed, 9 deletions(-) diff --git a/Lib/test/test_itertools.py b/Lib/test/test_itertools.py index d84fd85357ac89d..e285e8b4f74869e 100644 --- a/Lib/test/test_itertools.py +++ b/Lib/test/test_itertools.py @@ -2312,15 +2312,6 @@ def test_permutations_sizeof(self): ... else: ... return starmap(func, repeat(args, times)) ->>> def pairwise(iterable): -... "s -> (s0,s1), (s1,s2), (s2, s3), ..." -... a, b = tee(iterable) -... try: -... next(b) -... except StopIteration: -... pass -... return zip(a, b) - >>> def grouper(n, iterable, fillvalue=None): ... "grouper(3, 'ABCDEFG', 'x') --> ABC DEF Gxx" ... args = [iter(iterable)] * n From 404b46a7d06612ab3f47be31d08d54fd6bb476cc Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Sun, 29 Nov 2020 02:22:20 -0800 Subject: [PATCH 23/33] Add tests --- Lib/test/test_itertools.py | 34 ++++++++++++++++++++++++++++++++++ Modules/itertoolsmodule.c | 2 +- 2 files changed, 35 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_itertools.py b/Lib/test/test_itertools.py index e285e8b4f74869e..df2997e87d494cf 100644 --- a/Lib/test/test_itertools.py +++ b/Lib/test/test_itertools.py @@ -1024,6 +1024,25 @@ def run(r1, r2): self.assertEqual(next(it), (1, 2)) self.assertRaises(RuntimeError, next, it) + def test_pairwise(self): + self.assertEqual(list(pairwise('')), []) + self.assertEqual(list(pairwise('a')), []) + self.assertEqual(list(pairwise('ab')), + [('a', 'b')]), + self.assertEqual(list(pairwise('abcde')), + [('a', 'b'), ('b', 'c'), ('c', 'd'), ('d', 'e')]) + self.assertEqual(list(pairwise(range(10_000))), + list(zip(range(10_000), range(1, 10_000)))) + + with self.assertRaises(TypeError): + pairwise() # too few arguments + with self.assertRaises(TypeError): + pairwise('abc', 10) # too many arguments + with self.assertRaises(TypeError): + pairwise(iterable='abc') # keyword arguments + with self.assertRaises(TypeError): + pairwise(None) # non-iterable argument + def test_product(self): for args, result in [ ([], [()]), # zero iterables @@ -1787,6 +1806,10 @@ def test_islice(self): a = [] self.makecycle(islice([a]*2, None), a) + def test_pairwise(self): + a = [] + self.makecycle(pairwise([a]*5), a) + def test_permutations(self): a = [] self.makecycle(permutations([1,2,a,3], 3), a) @@ -1995,6 +2018,17 @@ def test_islice(self): self.assertRaises(TypeError, islice, N(s), 10) self.assertRaises(ZeroDivisionError, list, islice(E(s), 10)) + def test_pairwise(self): + for s in ("123", "", range(1000), ('do', 1.2), range(2000,2200,5)): + for g in (G, I, Ig, S, L, R): + seq = list(g(s)) + expected = list(zip(seq, seq[1:])) + actual = list(pairwise(g(s))) + self.assertEqual(actual, expected) + self.assertRaises(TypeError, pairwise, X(s)) + self.assertRaises(TypeError, pairwise, N(s)) + self.assertRaises(ZeroDivisionError, list, pairwise(E(s))) + def test_starmap(self): for s in (range(10), range(0), range(100), (7,11), range(20,50,5)): for g in (G, I, Ig, S, L, R): diff --git a/Modules/itertoolsmodule.c b/Modules/itertoolsmodule.c index 5b37341161e5f4c..956b1c7dcb429fb 100644 --- a/Modules/itertoolsmodule.c +++ b/Modules/itertoolsmodule.c @@ -79,12 +79,12 @@ pairwise_new_impl(PyTypeObject *type, PyObject *iterable) if (po == NULL) { return NULL; } + po->old = NULL; po->it = PyObject_GetIter(iterable); if (po->it == NULL) { Py_DECREF(po); return NULL; } - po->old = NULL; return (PyObject *)po; } From 5b1ebf6ec20b92814210f1c2c6b06b6606c874d1 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Sun, 29 Nov 2020 02:47:29 -0800 Subject: [PATCH 24/33] Tighten-up the code a bit --- Modules/itertoolsmodule.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/Modules/itertoolsmodule.c b/Modules/itertoolsmodule.c index 956b1c7dcb429fb..25e8cbf872a73bb 100644 --- a/Modules/itertoolsmodule.c +++ b/Modules/itertoolsmodule.c @@ -108,19 +108,20 @@ pairwise_traverse(pairwiseobject *po, visitproc visit, void *arg) static PyObject * pairwise_next(pairwiseobject *po) { + PyObject *it = po->it; PyObject *new, *result; - if (po->it == NULL) { + if (it == NULL) { return NULL; } if (po->old == NULL) { - po->old = PyIter_Next(po->it); + po->old = (*Py_TYPE(it)->tp_iternext)(it); if (po->old == NULL) { Py_CLEAR(po->it); return NULL; } } - new = PyIter_Next(po->it); + new = (*Py_TYPE(it)->tp_iternext)(it); if (new == NULL) { Py_CLEAR(po->it); Py_CLEAR(po->old); From 1d8bd37604f8b86442f4f389de2499e1eb9ae6e0 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Sun, 29 Nov 2020 03:19:56 -0800 Subject: [PATCH 25/33] Clean-up __new__ by putting the iterator test first --- Modules/itertoolsmodule.c | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/Modules/itertoolsmodule.c b/Modules/itertoolsmodule.c index 25e8cbf872a73bb..998315c794b97a1 100644 --- a/Modules/itertoolsmodule.c +++ b/Modules/itertoolsmodule.c @@ -73,18 +73,21 @@ static PyObject * pairwise_new_impl(PyTypeObject *type, PyObject *iterable) /*[clinic end generated code: output=9f0267062d384456 input=a1ad925c82e0e901]*/ { + PyObject *it; pairwiseobject *po; + it = PyObject_GetIter(iterable); + if (it == NULL) { + return NULL; + } + po = (pairwiseobject *)type->tp_alloc(type, 0); if (po == NULL) { + Py_DECREF(it); return NULL; } + po->it = it; po->old = NULL; - po->it = PyObject_GetIter(iterable); - if (po->it == NULL) { - Py_DECREF(po); - return NULL; - } return (PyObject *)po; } From bc53d6f50fd1d920401ff81e94a631929a0b91e1 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Sun, 29 Nov 2020 03:29:04 -0800 Subject: [PATCH 26/33] Don't lookup the old variable twice --- Modules/itertoolsmodule.c | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/Modules/itertoolsmodule.c b/Modules/itertoolsmodule.c index 998315c794b97a1..8c6e95c2e6db3be 100644 --- a/Modules/itertoolsmodule.c +++ b/Modules/itertoolsmodule.c @@ -112,14 +112,15 @@ static PyObject * pairwise_next(pairwiseobject *po) { PyObject *it = po->it; + PyObject *old = po->old; PyObject *new, *result; if (it == NULL) { return NULL; } - if (po->old == NULL) { - po->old = (*Py_TYPE(it)->tp_iternext)(it); - if (po->old == NULL) { + if (old == NULL) { + po->old = old = (*Py_TYPE(it)->tp_iternext)(it); + if (old == NULL) { Py_CLEAR(po->it); return NULL; } @@ -130,7 +131,7 @@ pairwise_next(pairwiseobject *po) Py_CLEAR(po->old); return NULL; } - result = PyTuple_Pack(2, po->old, new); + result = PyTuple_Pack(2, old, new); Py_SETREF(po->old, new); return result; } From 60bfd768cb49c9aec5d0ff920d66bd0fc4429e69 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Sun, 29 Nov 2020 03:36:31 -0800 Subject: [PATCH 27/33] Minor change in ordering --- Modules/itertoolsmodule.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Modules/itertoolsmodule.c b/Modules/itertoolsmodule.c index 8c6e95c2e6db3be..3e65b396bd35aa6 100644 --- a/Modules/itertoolsmodule.c +++ b/Modules/itertoolsmodule.c @@ -1,4 +1,5 @@ + #define PY_SSIZE_T_CLEAN #include "Python.h" #include "pycore_long.h" // _PyLong_GetZero() @@ -4827,6 +4828,7 @@ itertoolsmodule_exec(PyObject *m) &filterfalse_type, &count_type, &ziplongest_type, + &pairwise_type &permutations_type, &product_type, &repeat_type, @@ -4834,7 +4836,6 @@ itertoolsmodule_exec(PyObject *m) &_grouper_type, &tee_type, &teedataobject_type, - &pairwise_type }; Py_SET_TYPE(&teedataobject_type, &PyType_Type); From ff2244570b48c694962f3aab26e9602259177845 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Sun, 29 Nov 2020 03:37:33 -0800 Subject: [PATCH 28/33] Fix missing comma --- Modules/itertoolsmodule.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Modules/itertoolsmodule.c b/Modules/itertoolsmodule.c index 3e65b396bd35aa6..d0e1fb3af2868e9 100644 --- a/Modules/itertoolsmodule.c +++ b/Modules/itertoolsmodule.c @@ -4828,7 +4828,7 @@ itertoolsmodule_exec(PyObject *m) &filterfalse_type, &count_type, &ziplongest_type, - &pairwise_type + &pairwise_type, &permutations_type, &product_type, &repeat_type, From 7aefcd35cf5a7cbc1901eeac122deb9a78b66f8a Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Sun, 29 Nov 2020 03:38:38 -0800 Subject: [PATCH 29/33] Don't unnecessarily change a line in the original code --- Modules/itertoolsmodule.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Modules/itertoolsmodule.c b/Modules/itertoolsmodule.c index d0e1fb3af2868e9..9a905db3809f4e2 100644 --- a/Modules/itertoolsmodule.c +++ b/Modules/itertoolsmodule.c @@ -4835,7 +4835,7 @@ itertoolsmodule_exec(PyObject *m) &groupby_type, &_grouper_type, &tee_type, - &teedataobject_type, + &teedataobject_type }; Py_SET_TYPE(&teedataobject_type, &PyType_Type); From c56443a75d77607b9976d38d448107dc1a1ae40e Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Sun, 29 Nov 2020 11:00:33 -0800 Subject: [PATCH 30/33] Add todo comment --- Modules/itertoolsmodule.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Modules/itertoolsmodule.c b/Modules/itertoolsmodule.c index 9a905db3809f4e2..501b97dbc46c7e5 100644 --- a/Modules/itertoolsmodule.c +++ b/Modules/itertoolsmodule.c @@ -81,7 +81,6 @@ pairwise_new_impl(PyTypeObject *type, PyObject *iterable) if (it == NULL) { return NULL; } - po = (pairwiseobject *)type->tp_alloc(type, 0); if (po == NULL) { Py_DECREF(it); @@ -132,6 +131,7 @@ pairwise_next(pairwiseobject *po) Py_CLEAR(po->old); return NULL; } + /* Future optimization: Reuse the result tuple as we do in enumerate() */ result = PyTuple_Pack(2, old, new); Py_SETREF(po->old, new); return result; From 0020b6061af0870b8e74d6c11e7e2993cf2b4d68 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Sun, 29 Nov 2020 17:42:30 -0800 Subject: [PATCH 31/33] Update docstrings --- Modules/clinic/itertoolsmodule.c.h | 6 +++--- Modules/itertoolsmodule.c | 7 ++++--- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/Modules/clinic/itertoolsmodule.c.h b/Modules/clinic/itertoolsmodule.c.h index a382ac84e348ce4..82729eeb56bce9d 100644 --- a/Modules/clinic/itertoolsmodule.c.h +++ b/Modules/clinic/itertoolsmodule.c.h @@ -6,9 +6,9 @@ PyDoc_STRVAR(pairwise_new__doc__, "pairwise(iterable, /)\n" "--\n" "\n" -"Return a pairwise object.\n" +"Return an iterator of overlapping pairs taken from the input iterator.\n" "\n" -"\"s -> (s0,s1), (s1,s2), (s2, s3), ...\""); +" s -> (s0,s1), (s1,s2), (s2, s3), ..."); static PyObject * pairwise_new_impl(PyTypeObject *type, PyObject *iterable); @@ -658,4 +658,4 @@ itertools_count(PyTypeObject *type, PyObject *args, PyObject *kwargs) exit: return return_value; } -/*[clinic end generated code: output=540324a5db73ac98 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=889c4afc3b13574f input=a9049054013a1b77]*/ diff --git a/Modules/itertoolsmodule.c b/Modules/itertoolsmodule.c index 501b97dbc46c7e5..7144856c352f81f 100644 --- a/Modules/itertoolsmodule.c +++ b/Modules/itertoolsmodule.c @@ -64,15 +64,15 @@ typedef struct { itertools.pairwise.__new__ as pairwise_new iterable: object / -Return a pairwise object. +Return an iterator of overlapping pairs taken from the input iterator. -"s -> (s0,s1), (s1,s2), (s2, s3), ..." + s -> (s0,s1), (s1,s2), (s2, s3), ... [clinic start generated code]*/ static PyObject * pairwise_new_impl(PyTypeObject *type, PyObject *iterable) -/*[clinic end generated code: output=9f0267062d384456 input=a1ad925c82e0e901]*/ +/*[clinic end generated code: output=9f0267062d384456 input=6e7c3cddb431a8d6]*/ { PyObject *it; pairwiseobject *po; @@ -4799,6 +4799,7 @@ groupby(iterable[, keyfunc]) --> sub-iterators grouped by value of keyfunc(v)\n\ filterfalse(pred, seq) --> elements of seq where pred(elem) is False\n\ islice(seq, [start,] stop [, step]) --> elements from\n\ seq[start:stop:step]\n\ +pairwise(s) --> (s[0],s[1]), (s[1],s[2]), (s[2], s[3]), ...\n\ starmap(fun, seq) --> fun(*seq[0]), fun(*seq[1]), ...\n\ tee(it, n=2) --> (it1, it2 , ... itn) splits one iterator into n\n\ takewhile(pred, seq) --> seq[0], seq[1], until pred fails\n\ From 7927e1fdb9c6510d2cb3c544c1c36c67e6326ba6 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Sun, 29 Nov 2020 18:42:46 -0800 Subject: [PATCH 32/33] Add main docs --- Doc/library/itertools.rst | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/Doc/library/itertools.rst b/Doc/library/itertools.rst index 392a370b880f3a3..c5593ef93ce97c3 100644 --- a/Doc/library/itertools.rst +++ b/Doc/library/itertools.rst @@ -55,6 +55,7 @@ Iterator Arguments Results :func:`filterfalse` pred, seq elements of seq where pred(elem) is false ``filterfalse(lambda x: x%2, range(10)) --> 0 2 4 6 8`` :func:`groupby` iterable[, key] sub-iterators grouped by value of key(v) :func:`islice` seq, [start,] stop [, step] elements from seq[start:stop:step] ``islice('ABCDEFG', 2, None) --> C D E F G`` +:func:`pairwise` iterable (p[0], p[1]), (p[1], p[2]) ``pairwise('ABCDEFG') --> AB BC CD DE EF FG`` :func:`starmap` func, seq func(\*seq[0]), func(\*seq[1]), ... ``starmap(pow, [(2,5), (3,2), (10,3)]) --> 32 9 1000`` :func:`takewhile` pred, seq seq[0], seq[1], until pred fails ``takewhile(lambda x: x<5, [1,4,6,4,1]) --> 1 4`` :func:`tee` it, n it1, it2, ... itn splits one iterator into n @@ -475,6 +476,21 @@ loops that truncate the stream. If *start* is ``None``, then iteration starts at zero. If *step* is ``None``, then the step defaults to one. +.. function:: pairwise(iterable) + + Return successive overlapping pairs taken from the input *iterable*. + + The number of 2-tuples in output iterator will be one fewer than the number + of inputs. It will be empty if the input iterable has fewer than two values. + + Roughly equivalent to:: + + def pairwise(iterable): + # pairwise('ABCDEFG') --> AB BC CD DE EF FG + a, b = tee(iterable) + next(b, None) + return zip(a, b) + .. function:: permutations(iterable, r=None) From 5feeb72a29380bb25b878f9cb90dd4abfe90a29f Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Sun, 29 Nov 2020 22:43:49 -0800 Subject: [PATCH 33/33] Missing word --- Doc/library/itertools.rst | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Doc/library/itertools.rst b/Doc/library/itertools.rst index c5593ef93ce97c3..44728b42287bce2 100644 --- a/Doc/library/itertools.rst +++ b/Doc/library/itertools.rst @@ -480,8 +480,9 @@ loops that truncate the stream. Return successive overlapping pairs taken from the input *iterable*. - The number of 2-tuples in output iterator will be one fewer than the number - of inputs. It will be empty if the input iterable has fewer than two values. + The number of 2-tuples in the output iterator will be one fewer than the + number of inputs. It will be empty if the input iterable has fewer than + two values. Roughly equivalent to::