From 8ed52bf348e8755ea9b2f95ddf5d06f644873aaa Mon Sep 17 00:00:00 2001 From: nick evans Date: Mon, 25 Aug 2025 09:36:31 -0400 Subject: [PATCH] =?UTF-8?q?=E2=9A=A1=EF=B8=8F=20Don't=20memoize=20normaliz?= =?UTF-8?q?ed=20SequenceSet#string?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Not duplicating the data in `@tuples` and `@string` saves memory. For large sequence sets, this memory savings can be substantial. But this is a tradeoff: it saves time when the string is not used, but uses more time when the string is used more than once. Working with set operations can create many ephemeral sets, so avoiding unintentional string generation can save a lot of time. Also, by quickly scanning the entries after a string is parsed, we can bypass the merge algorithm for normalized strings. But this does cause a small penalty for non-normalized strings. **Please note:** It _is still possible_ to create a memoized string on a normalized SequenceSet with `#append`. For example: create a monotonically sorted SequenceSet with non-normal final entry, then call `#append` with an adjacently following entry. `#append` coalesces the final entry and converts it into normal form, but doesn't check whether the _preceding entries_ of the SequenceSet are normalized. -------------------------------------------------------------------- Results from benchmarks/sequence_set-normalize.yml There is still room for improvement here, because #normalize generates the normalized string for comparison rather than just reparse the string. ``` normal local: 19938.9 i/s v0.5.12: 2988.7 i/s - 6.67x slower frozen and normal local: 17011413.5 i/s v0.5.12: 3574.4 i/s - 4759.30x slower unsorted local: 19434.9 i/s v0.5.12: 2957.5 i/s - 6.57x slower abnormal local: 19835.9 i/s v0.5.12: 3037.1 i/s - 6.53x slower ``` -------------------------------------------------------------------- Results from benchmarks/sequence_set-new.yml Note that this benchmark doesn't use `SequenceSet::new`; it uses `SequenceSet::[]`, which freezes the result. In this case, the benchmark result differences are mostly driven by improved performance of `#freeze`. ``` n= 10 ints (sorted) local: 118753.9 i/s v0.5.12: 85411.4 i/s - 1.39x slower n= 10 string (sorted) v0.5.12: 123087.2 i/s local: 122746.3 i/s - 1.00x slower n= 10 ints (shuffled) local: 105919.2 i/s v0.5.12: 79294.5 i/s - 1.34x slower n= 10 string (shuffled) v0.5.12: 114826.6 i/s local: 108086.2 i/s - 1.06x slower n= 100 ints (sorted) local: 16418.4 i/s v0.5.12: 11864.2 i/s - 1.38x slower n= 100 string (sorted) local: 18161.7 i/s v0.5.12: 15219.3 i/s - 1.19x slower n= 100 ints (shuffled) local: 16640.1 i/s v0.5.12: 11815.8 i/s - 1.41x slower n= 100 string (shuffled) v0.5.12: 14755.8 i/s local: 14512.8 i/s - 1.02x slower n= 1,000 ints (sorted) local: 1722.2 i/s v0.5.12: 1229.0 i/s - 1.40x slower n= 1,000 string (sorted) local: 1862.1 i/s v0.5.12: 1543.2 i/s - 1.21x slower n= 1,000 ints (shuffled) local: 1684.9 i/s v0.5.12: 1252.3 i/s - 1.35x slower n= 1,000 string (shuffled) v0.5.12: 1467.3 i/s local: 1424.6 i/s - 1.03x slower n= 10,000 ints (sorted) local: 158.1 i/s v0.5.12: 127.9 i/s - 1.24x slower n= 10,000 string (sorted) local: 187.7 i/s v0.5.12: 143.4 i/s - 1.31x slower n= 10,000 ints (shuffled) local: 145.8 i/s v0.5.12: 114.5 i/s - 1.27x slower n= 10,000 string (shuffled) v0.5.12: 138.4 i/s local: 136.9 i/s - 1.01x slower n=100,000 ints (sorted) local: 14.9 i/s v0.5.12: 10.6 i/s - 1.40x slower n=100,000 string (sorted) local: 19.2 i/s v0.5.12: 14.0 i/s - 1.37x slower ``` The new code is ~1-6% slower for shuffled strings, but ~30-40% faster for sorted sets (note that unsorted non-string inputs create a sorted set). 📚 Update SequenceSet#normalize rdoc --- benchmarks/sequence_set-new.yml | 73 ++++++++++++++++++++++----- benchmarks/sequence_set-normalize.yml | 4 ++ lib/net/imap/sequence_set.rb | 59 ++++++++++++++++------ 3 files changed, 107 insertions(+), 29 deletions(-) diff --git a/benchmarks/sequence_set-new.yml b/benchmarks/sequence_set-new.yml index c7ecb2ca2..5a8f03706 100644 --- a/benchmarks/sequence_set-new.yml +++ b/benchmarks/sequence_set-new.yml @@ -9,8 +9,17 @@ prelude: | N_RAND = 100 def rand_nums(n, min: 1, max: (n * 1.25).to_i) = Array.new(n) { rand(1..max) } - def rand_entries(...) = SeqSet[rand_nums(...)].elements.shuffle - def rand_string(...) = SeqSet[rand_nums(...)].string.split(?,).shuffle.join(?,) + def rand_entries(...) = SeqSet[rand_nums(...)].elements + def rand_string(...) = SeqSet[rand_nums(...)].string + + def shuffle(inputs) + inputs.map! do + case _1 + in Array => elements then elements.shuffle + in String => string then string.split(?,).shuffle.join(?,) + end + end + end def build_string_inputs(n, n_rand, **) Array.new(n_rand) { rand_string(n, **) } @@ -35,51 +44,83 @@ prelude: | benchmark: - - name: n=10 ints + - name: n= 10 ints (sorted) prelude: inputs = build_int_inputs 10, N_RAND script: SeqSet[inputs[i = (i+1) % N_RAND]] - - name: n=10 string + - name: n= 10 string (sorted) prelude: inputs = build_string_inputs 10, N_RAND script: SeqSet[inputs[i = (i+1) % N_RAND]] - - name: n=100 ints + - name: n= 10 ints (shuffled) + prelude: inputs = build_int_inputs 10, N_RAND and shuffle inputs + script: SeqSet[inputs[i = (i+1) % N_RAND]] + + - name: n= 10 string (shuffled) + prelude: inputs = build_string_inputs 10, N_RAND and shuffle inputs + script: SeqSet[inputs[i = (i+1) % N_RAND]] + + - name: n= 100 ints (sorted) prelude: inputs = build_int_inputs 100, N_RAND script: SeqSet[inputs[i = (i+1) % N_RAND]] - - name: n=100 string + - name: n= 100 string (sorted) prelude: inputs = build_string_inputs 100, N_RAND script: SeqSet[inputs[i = (i+1) % N_RAND]] - - name: n=1000 ints + - name: n= 100 ints (shuffled) + prelude: inputs = build_int_inputs 100, N_RAND and shuffle inputs + script: SeqSet[inputs[i = (i+1) % N_RAND]] + + - name: n= 100 string (shuffled) + prelude: inputs = build_string_inputs 100, N_RAND and shuffle inputs + script: SeqSet[inputs[i = (i+1) % N_RAND]] + + - name: n= 1,000 ints (sorted) prelude: inputs = build_int_inputs 1000, N_RAND script: SeqSet[inputs[i = (i+1) % N_RAND]] - - name: n=1000 string + - name: n= 1,000 string (sorted) prelude: inputs = build_string_inputs 1000, N_RAND script: SeqSet[inputs[i = (i+1) % N_RAND]] - - name: n=10,000 ints + - name: n= 1,000 ints (shuffled) + prelude: inputs = build_int_inputs 1000, N_RAND and shuffle inputs + script: SeqSet[inputs[i = (i+1) % N_RAND]] + + - name: n= 1,000 string (shuffled) + prelude: inputs = build_string_inputs 1000, N_RAND and shuffle inputs + script: SeqSet[inputs[i = (i+1) % N_RAND]] + + - name: n= 10,000 ints (sorted) prelude: inputs = build_int_inputs 10_000, N_RAND script: SeqSet[inputs[i = (i+1) % N_RAND]] - - name: n=10,000 string + - name: n= 10,000 string (sorted) prelude: inputs = build_string_inputs 10_000, N_RAND script: SeqSet[inputs[i = (i+1) % N_RAND]] - - name: n=100,000 ints + - name: n= 10,000 ints (shuffled) + prelude: inputs = build_int_inputs 10_000, N_RAND and shuffle inputs + script: SeqSet[inputs[i = (i+1) % N_RAND]] + + - name: n= 10,000 string (shuffled) + prelude: inputs = build_string_inputs 10_000, N_RAND and shuffle inputs + script: SeqSet[inputs[i = (i+1) % N_RAND]] + + - name: n=100,000 ints (sorted) prelude: inputs = build_int_inputs 100_000, N_RAND / 2 script: SeqSet[inputs[i = (i+1) % N_RAND]] - - name: n=100,000 string + - name: n=100,000 string (sorted) prelude: inputs = build_string_inputs 100_000, N_RAND / 2 script: SeqSet[inputs[i = (i+1) % (N_RAND / 2)]] -# - name: n=1,000,000 ints +# - name: n=1,000,000 ints # prelude: inputs = build_int_inputs 1_000_000 # script: SeqSet[inputs[i = (i+1) % N_RAND]] -# - name: n=10,000,000 ints +# - name: n=10,000,000 ints # prelude: inputs = build_int_inputs 10_000_000 # script: SeqSet[inputs[i = (i+1) % N_RAND]] @@ -89,6 +130,10 @@ contexts: $LOAD_PATH.unshift "./lib" $allowed_to_profile = true # only profile local code require: false + - name: v0.5.12 + gems: + net-imap: 0.5.12 + require: false - name: v0.5.9 gems: net-imap: 0.5.9 diff --git a/benchmarks/sequence_set-normalize.yml b/benchmarks/sequence_set-normalize.yml index 6893552b5..7755381b7 100644 --- a/benchmarks/sequence_set-normalize.yml +++ b/benchmarks/sequence_set-normalize.yml @@ -99,6 +99,10 @@ benchmark: contexts: # n.b: can't use anything newer as the baseline: it's over 500x faster! + - name: v0.5.12 + gems: + net-imap: 0.5.12 + require: false - name: v0.5.9 gems: net-imap: 0.5.9 diff --git a/lib/net/imap/sequence_set.rb b/lib/net/imap/sequence_set.rb index 8ae4d6411..15fa68473 100644 --- a/lib/net/imap/sequence_set.rb +++ b/lib/net/imap/sequence_set.rb @@ -145,6 +145,10 @@ class IMAP # #entries and #elements are identical. Use #append to preserve #entries # order while modifying a set. # + # Non-normalized sets store both representations of the set, which can more + # than double memory usage. Very large sequence sets should avoid + # denormalizing methods (such as #append) unless order is significant. + # # == Using * # # \IMAP sequence sets may contain a special value "*", which @@ -586,7 +590,7 @@ def valid_string # the set is updated the string will be normalized. # # Related: #valid_string, #normalized_string, #to_s, #inspect - def string; @string ||= normalized_string if valid? end + def string; @string || normalized_string if valid? end # Returns an array with #normalized_string when valid and an empty array # otherwise. @@ -605,13 +609,18 @@ def string=(input) clear elsif (str = String.try_convert(input)) modifying! # short-circuit before parsing the string - tuples = str_to_tuples str - @tuples, @string = [], -str - tuples_add tuples + entries = each_parsed_entry(str).to_a + clear + if normalized_entries?(entries) + @tuples.replace entries.map!(&:minmax) + else + tuples_add entries.map!(&:minmax) + @string = -str + end else raise ArgumentError, "expected a string or nil, got #{input.class}" end - str + input end # Returns the \IMAP +sequence-set+ string representation, or an empty @@ -624,7 +633,6 @@ def to_s; string || "" end # Freezes and returns the set. A frozen SequenceSet is Ractor-safe. def freeze return self if frozen? - string @tuples.each(&:freeze).freeze super end @@ -971,7 +979,10 @@ def add(element) # set = Net::IMAP::SequenceSet.new("2,1,9:10") # set.append(11..12) # => Net::IMAP::SequenceSet("2,1,9:12") # - # See SequenceSet@Ordered+and+Normalized+sets. + # Non-normalized sets store the string in addition to an internal + # normalized uint32 set representation. This can more than double memory + # usage, so large sets should avoid using #append unless preserving order + # is required. See SequenceSet@Ordered+and+Normalized+sets. # # Related: #add, #merge, #union def append(entry) @@ -1685,10 +1696,11 @@ def xor!(other) merge(other).subtract(both) end - # Returns a new SequenceSet with a normalized string representation. + # Returns a SequenceSet with a normalized string representation: entries + # have been sorted, deduplicated, and coalesced, and all entries + # are in normal form. Returns +self+ for frozen normalized sets, and a + # normalized duplicate otherwise. # - # The returned set's #string is sorted and deduplicated. Adjacent or - # overlapping elements will be merged into a single larger range. # See SequenceSet@Ordered+and+Normalized+sets. # # Net::IMAP::SequenceSet["1:5,3:7,10:9,10:11"].normalize @@ -1696,9 +1708,8 @@ def xor!(other) # # Related: #normalize!, #normalized_string def normalize - str = normalized_string - return self if frozen? && str == string - remain_frozen dup.instance_exec { @string = str&.-@; self } + return self if frozen? && (@string.nil? || @string == normalized_string) + remain_frozen dup.normalize! end # Resets #string to be sorted, deduplicated, and coalesced. Returns @@ -1884,9 +1895,27 @@ def export_string_entries(entries) def tuple_to_str(tuple) tuple.uniq.map{ from_tuple_int _1 }.join(":") end def str_to_tuples(str) str.split(",", -1).map! { str_to_tuple _1 } end - def str_to_tuple(str) + def str_to_tuple(str) parse_string_entry(str).minmax end + + def parse_string_entry(str) raise DataFormatError, "invalid sequence set string" if str.empty? - str.split(":", 2).map! { to_tuple_int _1 }.minmax + str.split(":", 2).map! { to_tuple_int _1 } + end + + # yields validated but unsorted [num] or [num, num] + def each_parsed_entry(str) + return to_enum(__method__, str) unless block_given? + str&.split(",", -1) do |entry| yield parse_string_entry(entry) end + end + + def normalized_entries?(entries) + max = nil + entries.each do |first, last| + return false if last && last <= first # 1:1 or 2:1 + return false if max && first <= max + 1 # 2,1 or 1,1 or 1,2 + max = last || first + end + true end def include_tuple?((min, max)) range_gte_to(min)&.cover?(min..max) end