From dffa24e90594b9b103731182a77630a5576c4440 Mon Sep 17 00:00:00 2001 From: chilaxan Date: Tue, 24 Aug 2021 22:55:12 -0400 Subject: [PATCH 1/7] Fix rangeiter_reduce in rangeobject.c --- Objects/rangeobject.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Objects/rangeobject.c b/Objects/rangeobject.c index 3e05707b1cee69..5c3230d860f8f1 100644 --- a/Objects/rangeobject.c +++ b/Objects/rangeobject.c @@ -813,7 +813,7 @@ rangeiter_reduce(rangeiterobject *r, PyObject *Py_UNUSED(ignored)) if (range == NULL) goto err; /* return the result */ - return Py_BuildValue("N(N)i", _PyEval_GetBuiltinId(&PyId_iter), + return Py_BuildValue("N(N)l", _PyEval_GetBuiltinId(&PyId_iter), range, r->index); err: Py_XDECREF(start); From c880127b37f832da92314344f5f24966df4e6638 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Langa?= Date: Thu, 26 Aug 2021 15:51:32 +0200 Subject: [PATCH 2/7] Add tests --- Lib/test/test_range.py | 38 ++++++++++++++++++++------------------ 1 file changed, 20 insertions(+), 18 deletions(-) diff --git a/Lib/test/test_range.py b/Lib/test/test_range.py index 107c0e2e11c7ce..8d8f40ef020519 100644 --- a/Lib/test/test_range.py +++ b/Lib/test/test_range.py @@ -374,26 +374,28 @@ def test_pickling(self): list(r)) def test_iterator_pickling(self): - testcases = [(13,), (0, 11), (-22, 10), (20, 3, -1), - (13, 21, 3), (-2, 2, 2), (2**65, 2**65+2)] + testcases = [(13,), (0, 11), (-22, 10), (20, 3, -1), (13, 21, 3), + (-2, 2, 2), (2**31-3, 2**31-1), (2**33, 2**33+2), + (2**63-3, 2**63-1), (2**65, 2**65+2)] for proto in range(pickle.HIGHEST_PROTOCOL + 1): for t in testcases: - it = itorg = iter(range(*t)) - data = list(range(*t)) - - d = pickle.dumps(it, proto) - it = pickle.loads(d) - self.assertEqual(type(itorg), type(it)) - self.assertEqual(list(it), data) - - it = pickle.loads(d) - try: - next(it) - except StopIteration: - continue - d = pickle.dumps(it, proto) - it = pickle.loads(d) - self.assertEqual(list(it), data[1:]) + with self.subTest(proto=proto, t=t): + it = itorg = iter(range(*t)) + data = list(range(*t)) + + d = pickle.dumps(it, proto) + it = pickle.loads(d) + self.assertEqual(type(itorg), type(it)) + self.assertEqual(list(it), data) + + it = pickle.loads(d) + try: + next(it) + except StopIteration: + continue + d = pickle.dumps(it, proto) + it = pickle.loads(d) + self.assertEqual(list(it), data[1:]) def test_exhausted_iterator_pickling(self): for proto in range(pickle.HIGHEST_PROTOCOL + 1): From bf3f9449b81312647a03eff21f6c70f2f1232c09 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Langa?= Date: Thu, 26 Aug 2021 18:24:28 +0200 Subject: [PATCH 3/7] Add test for overflowing integer range index --- Lib/test/test_range.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/Lib/test/test_range.py b/Lib/test/test_range.py index 8d8f40ef020519..d141b7abac048c 100644 --- a/Lib/test/test_range.py +++ b/Lib/test/test_range.py @@ -397,6 +397,23 @@ def test_iterator_pickling(self): it = pickle.loads(d) self.assertEqual(list(it), data[1:]) + def test_iterator_pickling_overflowing_index(self): + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + with self.subTest(proto=proto): + it = itorg = iter(range(2**32 + 2)) + data = list(range(2**32 + 2)[2**32:]) + + d = pickle.dumps(it, proto) + it = pickle.loads(d) + self.assertEqual(type(itorg), type(it)) + + it = pickle.loads(d) + it.__setstate__(2**32 + 1) # undocumented way to set r->index + d = pickle.dumps(it, proto) + it = pickle.loads(d) + self.assertEqual(list(it), data[1:]) + + def test_exhausted_iterator_pickling(self): for proto in range(pickle.HIGHEST_PROTOCOL + 1): r = range(2**65, 2**65+2) From e29696e7c4c333b41edc51afb963c8c2239bc22d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Langa?= Date: Thu, 26 Aug 2021 18:46:15 +0200 Subject: [PATCH 4/7] add Blurb --- .../Core and Builtins/2021-08-26-18-44-03.bpo-45018.pu8H9L.rst | 1 + 1 file changed, 1 insertion(+) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2021-08-26-18-44-03.bpo-45018.pu8H9L.rst diff --git a/Misc/NEWS.d/next/Core and Builtins/2021-08-26-18-44-03.bpo-45018.pu8H9L.rst b/Misc/NEWS.d/next/Core and Builtins/2021-08-26-18-44-03.bpo-45018.pu8H9L.rst new file mode 100644 index 00000000000000..5bf13ef06f34c3 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2021-08-26-18-44-03.bpo-45018.pu8H9L.rst @@ -0,0 +1 @@ +Fixed pickling of range iterators that iterated for over 2**32 times. From 2df71773b80ff5037b7bac35a0c5d1b727f3f4cc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Langa?= Date: Thu, 26 Aug 2021 20:13:34 +0200 Subject: [PATCH 5/7] Use a shorter test per Serhiy's review --- Lib/test/test_range.py | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/Lib/test/test_range.py b/Lib/test/test_range.py index d141b7abac048c..7ed2f22b709658 100644 --- a/Lib/test/test_range.py +++ b/Lib/test/test_range.py @@ -400,19 +400,11 @@ def test_iterator_pickling(self): def test_iterator_pickling_overflowing_index(self): for proto in range(pickle.HIGHEST_PROTOCOL + 1): with self.subTest(proto=proto): - it = itorg = iter(range(2**32 + 2)) - data = list(range(2**32 + 2)[2**32:]) - - d = pickle.dumps(it, proto) - it = pickle.loads(d) - self.assertEqual(type(itorg), type(it)) - - it = pickle.loads(d) + it = iter(range(2**32 + 2)) it.__setstate__(2**32 + 1) # undocumented way to set r->index d = pickle.dumps(it, proto) it = pickle.loads(d) - self.assertEqual(list(it), data[1:]) - + self.assertEqual(next(it), 2**32 + 1) def test_exhausted_iterator_pickling(self): for proto in range(pickle.HIGHEST_PROTOCOL + 1): From 72410686ef20f67e769ce4775e9593dd146006d6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Langa?= Date: Thu, 26 Aug 2021 20:16:01 +0200 Subject: [PATCH 6/7] Confirm __setstate__ worked by checking __reduce__ --- Lib/test/test_range.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Lib/test/test_range.py b/Lib/test/test_range.py index 7ed2f22b709658..0a2f115351d30d 100644 --- a/Lib/test/test_range.py +++ b/Lib/test/test_range.py @@ -401,7 +401,11 @@ def test_iterator_pickling_overflowing_index(self): for proto in range(pickle.HIGHEST_PROTOCOL + 1): with self.subTest(proto=proto): it = iter(range(2**32 + 2)) + _, _, idx = it.__reduce__() + self.assertEqual(idx, 0) it.__setstate__(2**32 + 1) # undocumented way to set r->index + _, _, idx = it.__reduce__() + self.assertEqual(idx, 2**32) d = pickle.dumps(it, proto) it = pickle.loads(d) self.assertEqual(next(it), 2**32 + 1) From 2d5783485bab25c58a95248ec035fe7f6d610fb5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Langa?= Date: Thu, 26 Aug 2021 20:33:06 +0200 Subject: [PATCH 7/7] =?UTF-8?q?I=20got=20overly=20excited=20with=20the=20a?= =?UTF-8?q?dditional=20asserts=20=F0=9F=A4=A6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Lib/test/test_range.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_range.py b/Lib/test/test_range.py index 0a2f115351d30d..897162b2b17457 100644 --- a/Lib/test/test_range.py +++ b/Lib/test/test_range.py @@ -405,7 +405,7 @@ def test_iterator_pickling_overflowing_index(self): self.assertEqual(idx, 0) it.__setstate__(2**32 + 1) # undocumented way to set r->index _, _, idx = it.__reduce__() - self.assertEqual(idx, 2**32) + self.assertEqual(idx, 2**32 + 1) d = pickle.dumps(it, proto) it = pickle.loads(d) self.assertEqual(next(it), 2**32 + 1)