From 8943e75c24eaa00a61d36c0eaae21c3693b0b176 Mon Sep 17 00:00:00 2001 From: "Brock A. Martin" Date: Tue, 20 Oct 2020 09:54:39 -0500 Subject: [PATCH 1/5] Fix update() ordering to be more consistent with add() ordering --- sortedcontainers/sortedlist.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/sortedcontainers/sortedlist.py b/sortedcontainers/sortedlist.py index 3b3cd07b..082d8033 100644 --- a/sortedcontainers/sortedlist.py +++ b/sortedcontainers/sortedlist.py @@ -339,7 +339,8 @@ def update(self, iterable): if _maxes: if len(values) * 4 >= self._len: - values.extend(chain.from_iterable(_lists)) + # prepend existing elements to new ones + values[0:0] = chain.from_iterable(_lists) values.sort() self._clear() else: @@ -1878,7 +1879,8 @@ def update(self, iterable): if _maxes: if len(values) * 4 >= self._len: - values.extend(chain.from_iterable(_lists)) + # prepend existing elements to new ones + values[0:0] = chain.from_iterable(_lists) values.sort(key=self._key) self._clear() else: From 2c46e4765813bb46998619c2458db89f2c9ef903 Mon Sep 17 00:00:00 2001 From: "Brock A. Martin" Date: Tue, 20 Oct 2020 10:41:56 -0500 Subject: [PATCH 2/5] Add tests for update order consistency --- tests/test_coverage_sortedkeylist_modulo.py | 13 +++++++++++++ tests/test_coverage_sortedkeylist_negate.py | 14 +++++++++++++- 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/tests/test_coverage_sortedkeylist_modulo.py b/tests/test_coverage_sortedkeylist_modulo.py index 1916054f..fc612a07 100644 --- a/tests/test_coverage_sortedkeylist_modulo.py +++ b/tests/test_coverage_sortedkeylist_modulo.py @@ -5,6 +5,7 @@ import random from .context import sortedcontainers from sortedcontainers import SortedList, SortedKeyList +from itertools import chain, repeat import pytest if hexversion < 0x03000000: @@ -107,6 +108,18 @@ def test_update(): assert len(slt) == 11000 slt._check() +def test_update_order_consistency(): + slt = SortedKeyList(key=lambda x: x[0]) + + it1 = list(zip(repeat(0), range(4))) + it2 = list(zip(repeat(0), range(5))) + + slt.update(it1) + slt.update(it2) + slt._check() + + assert all(tup[0] == tup[1] for tup in zip(slt, chain(it1, it2))) + def test_contains(): slt = SortedKeyList(key=modulo) slt._reset(7) diff --git a/tests/test_coverage_sortedkeylist_negate.py b/tests/test_coverage_sortedkeylist_negate.py index 6c0287f8..2bacfd7f 100644 --- a/tests/test_coverage_sortedkeylist_negate.py +++ b/tests/test_coverage_sortedkeylist_negate.py @@ -5,7 +5,7 @@ import random from .context import sortedcontainers from sortedcontainers import SortedKeyList, SortedListWithKey -from itertools import chain +from itertools import chain, repeat import pytest if hexversion < 0x03000000: @@ -84,6 +84,18 @@ def test_update(): values = sorted((val for val in chain(range(100), range(1000), range(10000))), key=negate) assert all(tup[0] == tup[1] for tup in zip(slt, values)) +def test_update_order_consistency(): + slt = SortedKeyList(key=lambda x: x[0]) + + it1 = list(zip(repeat(0), range(4))) + it2 = list(zip(repeat(0), range(5))) + + slt.update(it1) + slt.update(it2) + slt._check() + + assert all(tup[0] == tup[1] for tup in zip(slt, chain(it1, it2))) + def test_contains(): slt = SortedKeyList(key=negate) assert 0 not in slt From 85b3f1b9cc2b666ffd02d9e09aa5fefbb8d585d9 Mon Sep 17 00:00:00 2001 From: "Brock A. Martin" Date: Thu, 22 Oct 2020 08:54:51 -0500 Subject: [PATCH 3/5] Fix update order consistency test to use `modulo` key and to compare internally against using the `add()` method --- tests/test_coverage_sortedkeylist_modulo.py | 41 +++++++++++++++++---- 1 file changed, 34 insertions(+), 7 deletions(-) diff --git a/tests/test_coverage_sortedkeylist_modulo.py b/tests/test_coverage_sortedkeylist_modulo.py index fc612a07..5d2cf740 100644 --- a/tests/test_coverage_sortedkeylist_modulo.py +++ b/tests/test_coverage_sortedkeylist_modulo.py @@ -109,16 +109,43 @@ def test_update(): slt._check() def test_update_order_consistency(): - slt = SortedKeyList(key=lambda x: x[0]) + def modulo_el0(tup): + return tup[0] % 10 - it1 = list(zip(repeat(0), range(4))) - it2 = list(zip(repeat(0), range(5))) + slt1 = SortedKeyList(key=modulo_el0) + slt2 = SortedKeyList(key=modulo_el0) - slt.update(it1) - slt.update(it2) - slt._check() + def add_from_iterable(slt, it): + for item in it: + slt.add(item) + + def add_from_all_iterables(slt, its): + for it in its: + add_from_iterable(slt, it) + + def update_from_all_iterables(slt, its): + for it in its: + slt.update(it) + + # the following iterators are set up (from large to small) such that they + # attempt to force the two kinds of internal update logic (extending upon + # the incoming iterable or appending to the existing elements by use of + # `add()`) + it1 = list(zip(repeat(0), range(5))) + it2 = list(zip(repeat(0), range(4))) + it3 = list(zip(repeat(0), range(3))) + it4 = list(zip(repeat(0), range(2))) + it5 = list(zip(repeat(0), range(1))) + + it12345 = [it1, it2, it3, it4, it5] + + add_from_all_iterables(slt1, it12345) + update_from_all_iterables(slt2, it12345) + + slt1._check() + slt2._check() - assert all(tup[0] == tup[1] for tup in zip(slt, chain(it1, it2))) + assert all(tup[0] == tup[1] for tup in zip(slt1, slt2)) def test_contains(): slt = SortedKeyList(key=modulo) From 0aaede2f27e8dbc4fca61276ddc06595010e50f6 Mon Sep 17 00:00:00 2001 From: "Brock A. Martin" Date: Thu, 22 Oct 2020 08:55:17 -0500 Subject: [PATCH 4/5] Fix update order consistency test to use `negate` key and to compare internally against using the `add()` method --- tests/test_coverage_sortedkeylist_negate.py | 41 +++++++++++++++++---- 1 file changed, 34 insertions(+), 7 deletions(-) diff --git a/tests/test_coverage_sortedkeylist_negate.py b/tests/test_coverage_sortedkeylist_negate.py index 2bacfd7f..dea380db 100644 --- a/tests/test_coverage_sortedkeylist_negate.py +++ b/tests/test_coverage_sortedkeylist_negate.py @@ -85,16 +85,43 @@ def test_update(): assert all(tup[0] == tup[1] for tup in zip(slt, values)) def test_update_order_consistency(): - slt = SortedKeyList(key=lambda x: x[0]) + def negate_el0(tup): + return -tup[0] - it1 = list(zip(repeat(0), range(4))) - it2 = list(zip(repeat(0), range(5))) + slt1 = SortedKeyList(key=negate_el0) + slt2 = SortedKeyList(key=negate_el0) - slt.update(it1) - slt.update(it2) - slt._check() + def add_from_iterable(slt, it): + for item in it: + slt.add(item) + + def add_from_all_iterables(slt, its): + for it in its: + add_from_iterable(slt, it) + + def update_from_all_iterables(slt, its): + for it in its: + slt.update(it) + + # the following iterators are set up (from large to small) such that they + # attempt to force the two kinds of internal update logic (extending upon + # the incoming iterable or appending to the existing elements by use of + # `add()`) + it1 = list(zip(repeat(0), range(5))) + it2 = list(zip(repeat(0), range(4))) + it3 = list(zip(repeat(0), range(3))) + it4 = list(zip(repeat(0), range(2))) + it5 = list(zip(repeat(0), range(1))) + + it12345 = [it1, it2, it3, it4, it5] + + add_from_all_iterables(slt1, it12345) + update_from_all_iterables(slt2, it12345) + + slt1._check() + slt2._check() - assert all(tup[0] == tup[1] for tup in zip(slt, chain(it1, it2))) + assert all(tup[0] == tup[1] for tup in zip(slt1, slt2)) def test_contains(): slt = SortedKeyList(key=negate) From 80a1ce9f8b353918974a0ca7ffdfb32c43821026 Mon Sep 17 00:00:00 2001 From: "Brock A. Martin" Date: Wed, 28 Oct 2020 09:32:32 -0500 Subject: [PATCH 5/5] Improve performance by using `reduce`/`iadd` to construct `values` list --- sortedcontainers/sortedlist.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/sortedcontainers/sortedlist.py b/sortedcontainers/sortedlist.py index 082d8033..e3b58eb9 100644 --- a/sortedcontainers/sortedlist.py +++ b/sortedcontainers/sortedlist.py @@ -339,8 +339,8 @@ def update(self, iterable): if _maxes: if len(values) * 4 >= self._len: - # prepend existing elements to new ones - values[0:0] = chain.from_iterable(_lists) + _lists.append(values) + values = reduce(iadd, _lists, []) values.sort() self._clear() else: @@ -1879,8 +1879,8 @@ def update(self, iterable): if _maxes: if len(values) * 4 >= self._len: - # prepend existing elements to new ones - values[0:0] = chain.from_iterable(_lists) + _lists.append(values) + values = reduce(iadd, _lists, []) values.sort(key=self._key) self._clear() else: