From 8b81faf14990bf864796df4223f96ee86944abcd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Per=20=C3=96stlund?= Date: Wed, 18 Nov 2020 14:01:59 +0100 Subject: [PATCH] Data structures improvements - New hash table implementation UnorderedMap. - Small updates to UnorderedSet. - Some more functions for Vector. --- OMCompiler/Compiler/Util/UnorderedMap.mo | 548 ++++++++++++++++++ OMCompiler/Compiler/Util/UnorderedSet.mo | 18 +- OMCompiler/Compiler/Util/Vector.mo | 114 +++- .../Compiler/boot/LoadCompilerSources.mos | 1 + 4 files changed, 671 insertions(+), 10 deletions(-) create mode 100644 OMCompiler/Compiler/Util/UnorderedMap.mo diff --git a/OMCompiler/Compiler/Util/UnorderedMap.mo b/OMCompiler/Compiler/Util/UnorderedMap.mo new file mode 100644 index 00000000000..8d26291c652 --- /dev/null +++ b/OMCompiler/Compiler/Util/UnorderedMap.mo @@ -0,0 +1,548 @@ +/* + * This file is part of OpenModelica. + * + * Copyright (c) 1998-CurrentYear, Open Source Modelica Consortium (OSMC), + * c/o Linköpings universitet, Department of Computer and Information Science, + * SE-58183 Linköping, Sweden. + * + * All rights reserved. + * + * THIS PROGRAM IS PROVIDED UNDER THE TERMS OF GPL VERSION 3 LICENSE OR + * THIS OSMC PUBLIC LICENSE (OSMC-PL) VERSION 1.2. + * ANY USE, REPRODUCTION OR DISTRIBUTION OF THIS PROGRAM CONSTITUTES + * RECIPIENT'S ACCEPTANCE OF THE OSMC PUBLIC LICENSE OR THE GPL VERSION 3, + * ACCORDING TO RECIPIENTS CHOICE. + * + * The OpenModelica software and the Open Source Modelica + * Consortium (OSMC) Public License (OSMC-PL) are obtained + * from OSMC, either from the above address, + * from the URLs: http://www.ida.liu.se/projects/OpenModelica or + * http://www.openmodelica.org, and in the OpenModelica distribution. + * GNU version 3 is obtained from: http://www.gnu.org/copyleft/gpl.html. + * + * This program is distributed WITHOUT ANY WARRANTY; without + * even the implied warranty of MERCHANTABILITY or FITNESS + * FOR A PARTICULAR PURPOSE, EXCEPT AS EXPRESSLY SET FORTH + * IN THE BY RECIPIENT SELECTED SUBSIDIARY LICENSE CONDITIONS OF OSMC-PL. + * + * See the full OSMC Public License conditions for more details. + * + */ + +encapsulated uniontype UnorderedMap + + import Vector; + +protected + import List; + import MetaModelica.Dangerous.*; + import Util; + +public + partial function Hash + input K key; + input Integer mod; + output Integer hash; + end Hash; + + partial function KeyEq + input K key1; + input K key2; + output Boolean equal; + end KeyEq; + + record UNORDERED_MAP + Vector> buckets; + Vector keys; + Vector values; + Hash hashFn; + KeyEq eqFn; + end UNORDERED_MAP; + + function new + "Creates a new map given a hash function, key equality function, and + optional desired bucket count. An appropriate bucket count is + Util.nextPrime(number of elements that will be added), but starting with a + low bucket count is also fine if the number of elements is unknown since + the map rehashes as needed." + input Hash hash; + input KeyEq keyEq; + input Integer bucketCount = 1; + output UnorderedMap map; + protected + K kdummy = kdummy; + V vdummy = vdummy; + algorithm + map := UNORDERED_MAP( + Vector.newFill(bucketCount, {}), + Vector.new(kdummy), + Vector.new(vdummy), + hash, + keyEq + ); + end new; + + function copy + "Returns a copy of the map." + input UnorderedMap map; + output UnorderedMap outMap; + algorithm + outMap := UNORDERED_MAP( + Vector.copy(map.buckets), + Vector.copy(map.keys), + Vector.copy(map.values), + map.hashFn, + map.eqFn + ); + end copy; + + function deepCopy + "Returns a deep copy of the map using the given copy function for values." + input UnorderedMap map; + input CopyFn fn; + output UnorderedMap outMap; + + partial function CopyFn + input output V value; + end CopyFn; + algorithm + outMap := UNORDERED_MAP( + Vector.copy(map.buckets), + Vector.copy(map.keys), + Vector.deepCopy(map.values, fn), + map.hashFn, + map.eqFn + ); + end deepCopy; + + function add + "Adds a key and associated value to the map, or updated the value if the key + already exists in the map. Might trigger a rehash." + input K key; + input V value; + input UnorderedMap map; + protected + Integer index, hash; + algorithm + (index, hash) := find(key, map); + + if index > 0 then + Vector.update(map.values, index, value); + else + addEntry(key, value, hash, map); + end if; + end add; + + function addNew + "Adds a key and associated value to the map without checking if it already + exists. Faster than add since it doesn't need to check if the key exists, + but will lead to duplicate keys if it actually does exist in the map + already. Might trigger a rehash." + input K key; + input V value; + input UnorderedMap map; + protected + Hash hashfn = map.hashFn; + Integer hash; + algorithm + hash := hashfn(key, Vector.size(map.buckets)); + addEntry(key, value, hash, map); + end addNew; + + function addUnique + "Adds a key and associated value to the map, but fails if the key already + exists. Might trigger a rehash." + input K key; + input V value; + input UnorderedMap map; + protected + Integer index, hash; + algorithm + (index, hash) := find(key, map); + true := index > 0; + addEntry(key, value, hash, map); + end addUnique; + + function addUpdate + "Adds a key and associated value to the map, where the value is generated by + calling the given function. If the key already exists the function is given + the old value. + + This function can be used to e.g. append to an existing value or create a + new value if none exists. This is faster than trying to fetch the value, + updating it and then readding it, since the key only needs to be hashed + once." + input K key; + input UpdateFn fn; + input UnorderedMap map; + + partial function UpdateFn + input Option oldValue; + output V value; + end UpdateFn; + protected + Integer index, hash, bucket_count; + V value; + algorithm + (index, hash) := find(key, map); + + if index > 0 then + bucket_count := Vector.size(map.buckets); + value := fn(SOME(Vector.getNoBounds(map.values, index))); + Vector.updateNoBounds(map.values, index, value); + else + value := fn(NONE()); + addEntry(key, value, hash, map); + end if; + end addUpdate; + + function remove + "Removes a key from the map. Returns true if the key existed in the map and + was removed, or false if the key did not exist in the map. This function is + O(N) since it will remove the key/value from the key/value arrays. + + Will not trigger a rehash, so rehash must be called manually if shrinking + the map is desirable (probably not a good idea unless the load factor is + very low, i.e. less than 0.25 or so)." + input K key; + input UnorderedMap map; + output Boolean removed; + protected + Integer hash, index; + list bucket; + algorithm + (index, hash) := find(key, map); + removed := index > 0; + + // Key didn't exist in the map, do nothing. + if not removed then + return; + end if; + + // Remove the index from the bucket. + bucket := Vector.get(map.buckets, hash + 1); + bucket := List.deleteMemberOnTrue(index, bucket, intEq); + Vector.updateNoBounds(map.buckets, hash + 1, bucket); + + // Remove the key/value from the arrays. + Vector.remove(map.keys, index); + Vector.remove(map.values, index); + end remove; + + function get + "Returns SOME(value) if the given key has an associated value in the map, + otherwise NONE()." + input K key; + input UnorderedMap map; + output Option value; + protected + Integer index; + algorithm + index := find(key, map); + value := if index > 0 then SOME(Vector.getNoBounds(map.values, index)) else NONE(); + end get; + + function getKey + "Returns SOME(key) if the key exists in the map, otherwise NONE()." + input K key; + input UnorderedMap map; + output Option outKey; + protected + Integer index; + algorithm + index := find(key, map); + outKey := if index > 0 then SOME(Vector.getNoBounds(map.keys, index)) else NONE(); + end getKey; + + function contains + "Returns whether the given key exists in the map or not." + input K key; + input UnorderedMap map; + output Boolean res; + algorithm + res := find(key, map) > 0; + end contains; + + function first + "Returns the 'first' element in the map, or fails if the map is empty." + input UnorderedMap map; + output V value; + algorithm + value := Vector.get(map.values, 1); + end first; + + function keyList + "Returns the keys as a list." + input UnorderedMap map; + output list keys; + algorithm + keys := Vector.toList(map.keys); + end keyList; + + function valueList + "Returns the values as a list." + input UnorderedMap map; + output list values; + algorithm + values := Vector.toList(map.values); + end valueList; + + function keyArray + "Returns the keys as an array." + input UnorderedMap map; + output array keys; + algorithm + keys := Vector.toArray(map.keys); + end keyArray; + + function valueArray + "Returns the values as an array." + input UnorderedMap map; + output array values; + algorithm + values := Vector.toArray(map.values); + end valueArray; + + function keyVector + "Returns the keys as a Vector." + input UnorderedMap map; + output Vector keys; + algorithm + keys := Vector.copy(map.keys); + end keyVector; + + function valueVector + "Returns the values as a Vector." + input UnorderedMap map; + output Vector values; + algorithm + values := Vector.copy(map.values); + end valueVector; + + function fold + "Folds over the keys in the map." + input UnorderedMap map; + input FoldFn fn; + input output FT arg; + + partial function FoldFn + input K value; + input output FT arg; + end FoldFn; + algorithm + arg := Vector.fold(map.values, fn, arg); + end fold; + + function map + "Applies a function to each value in the given map and returns a copy of the + map with the new values." + input UnorderedMap map; + input MapFn fn; + output UnorderedMap outMap; + + partial function MapFn + input V value; + output OT outValue; + end MapFn; + protected + Vector new_values; + algorithm + new_values := Vector.map(map.values, fn); + outMap := UNORDERED_MAP( + Vector.copy(map.buckets), + Vector.copy(map.keys), + new_values, + map.hashFn, + map.eqFn + ); + end map; + + function apply + "Replaces each value in the given map with the result of the given function + when applied to each value." + input UnorderedMap map; + input ApplyFn fn; + + partial function ApplyFn + input output V value; + end ApplyFn; + algorithm + Vector.apply(map.values, fn); + end apply; + + function all + "Returns true if the given function returns true for all values in the map, + otherwise false." + input UnorderedMap map; + input PredFn fn; + output Boolean res; + + partial function PredFn + input V value; + output Boolean res; + end PredFn; + algorithm + res := Vector.all(map.values, fn); + end all; + + function any + "Returns true if the given function returns true for any value in the map, + otherwise false." + input UnorderedMap map; + input PredFn fn; + output Boolean res; + + partial function PredFn + input V value; + output Boolean res; + end PredFn; + algorithm + res := Vector.any(map.values, fn); + end any; + + function none + "Returns true if the given function returns true for none of the values in + the map, otherwise false." + input UnorderedMap map; + input PredFn fn; + output Boolean res; + + partial function PredFn + input V value; + output Boolean res; + end PredFn; + algorithm + res := Vector.none(map.values, fn); + end none; + + function size + "Returns the number of elements the map contains." + input UnorderedMap map; + output Integer size = Vector.size(map.keys); + end size; + + function isEmpty + "Returns whether the map is empty or not." + input UnorderedMap map; + output Boolean empty = Vector.isEmpty(map.keys); + end isEmpty; + + function bucketCount + "Returns the number of buckets in the map." + input UnorderedMap map; + output Integer count = Vector.size(map.buckets); + end bucketCount; + + function loadFactor + "Returns the load factor, defined as the number of entries divided by the + number of buckets." + input UnorderedMap map; + output Real load = intReal(Vector.size(map.keys)) / Vector.size(map.buckets); + end loadFactor; + + function rehash + "Changes the number of buckets to an appropriate number based on the number + of elements in the map and rehashes all the keys." + input UnorderedMap map; + protected + Vector keys = map.keys; + Vector> buckets = map.buckets; + Integer bucket_count, bucket_id; + Hash hashfn = map.hashFn; + algorithm + // Clear the buckets. + Vector.clear(buckets); + + // Change the number of buckets for a load factor of about 0.5. + bucket_count := Util.nextPrime(Vector.size(keys) * 2); + Vector.resize(buckets, bucket_count, {}); + + // Rehash all the keys and refill the buckets. + for i in 1:Vector.size(map.keys) loop + bucket_id := hashfn(Vector.get(keys, i), bucket_count) + 1; + Vector.updateNoBounds(buckets, bucket_id, i :: Vector.getNoBounds(buckets, bucket_id)); + end for; + end rehash; + + function toString + "Returns a string representation of the map." + input UnorderedMap map; + input KeyStringFn keyStringFn; + input ValueStringFn valueStringFn; + input String delimiter = "\n"; + output String str; + + partial function KeyStringFn + input K key; + output String str; + end KeyStringFn; + + partial function ValueStringFn + input V value; + output String str; + end ValueStringFn; + protected + list strl = {}; + Vector keys = map.keys; + Vector values = map.values; + algorithm + for i in Vector.size(keys):-1:1 loop + strl := "(" + keyStringFn(Vector.get(keys, i)) + ", " + + valueStringFn(Vector.get(values, i)) + ")" :: strl; + end for; + + str := stringDelimitList(strl, delimiter); + end toString; + +protected + function find + "Returns the array index of the given key (or -1 if the key isn't in the map) + and the key's hash." + input K key; + input UnorderedMap map; + output Integer index = -1; + output Integer hash; + protected + Hash hashfn = map.hashFn; + KeyEq eqfn = map.eqFn; + list bucket; + algorithm + hash := hashfn(key, Vector.size(map.buckets)); + bucket := Vector.get(map.buckets, hash + 1); + + for i in bucket loop + if eqfn(key, Vector.getNoBounds(map.keys, i)) then + index := i; + break; + end if; + end for; + end find; + + function addEntry + "Adds a key and value to the map given the key's hash." + input K key; + input V value; + input Integer hash; + input UnorderedMap map; + protected + Vector> buckets = map.buckets; + Integer h; + Hash hashfn; + algorithm + // Add the key/value to the key/value arrays. + Vector.push(map.keys, key); + Vector.push(map.values, value); + + if loadFactor(map) > 1 then + // Rehash if the load factor is too high to keep performance up. This + // rehashes all the keys, including the one we added above. + rehash(map); + else + // Otherwise add the index of the key/value to the correct bucket. + Vector.update(buckets, hash + 1, + Vector.size(map.keys) :: Vector.get(buckets, hash + 1)); + end if; + end addEntry; + +annotation(__OpenModelica_Interface="util"); +end UnorderedMap; diff --git a/OMCompiler/Compiler/Util/UnorderedSet.mo b/OMCompiler/Compiler/Util/UnorderedSet.mo index 845993a3781..6cf7c05892b 100644 --- a/OMCompiler/Compiler/Util/UnorderedSet.mo +++ b/OMCompiler/Compiler/Util/UnorderedSet.mo @@ -137,11 +137,15 @@ public end addUnique; function remove - "Removes a key from the set. Will not trigger a rehash, so rehash must to be - called manually if shrinking the set is desirable (probably not a good idea - unless the load factor is very low, i.e. less than 0.25 or so)." + "Removes a key from the set. Returns true if the key existed in the set and + was removed, or false if the key did not exist in the set. + + Will not trigger a rehash, so rehash must be called manually if shrinking + the set is desirable (probably not a good idea unless the load factor is + very low, i.e. less than 0.25 or so)." input T key; input UnorderedSet set; + output Boolean removed; protected array> buckets = Mutable.access(set.buckets); Hash hashfn = set.hashFn; @@ -154,8 +158,9 @@ public bucket := arrayGet(buckets, hash + 1); (bucket, okey) := List.deleteMemberOnTrue(key, bucket, eqfn); + removed := isSome(okey); - if isSome(okey) then + if removed then arrayUpdateNoBoundsChecking(buckets, hash + 1, bucket); Mutable.update(set.size, Mutable.access(set.size) - 1); end if; @@ -436,15 +441,12 @@ public protected function find "Tries to find a key in the set, returning the key as an option, and the - key's hash. If the key isn't in the set it returns NONE() and -1 as index, - but the correct hash is always returned." + key's hash." input T key; input UnorderedSet set; output Option outKey = NONE(); output Integer hash; protected - T k; - Integer hash_id, i; Hash hashfn = set.hashFn; KeyEq eqfn = set.eqFn; array> buckets = Mutable.access(set.buckets); diff --git a/OMCompiler/Compiler/Util/Vector.mo b/OMCompiler/Compiler/Util/Vector.mo index 733755c9cbd..4245e7687ad 100644 --- a/OMCompiler/Compiler/Util/Vector.mo +++ b/OMCompiler/Compiler/Util/Vector.mo @@ -226,17 +226,33 @@ public end appendArray; function pop - "Removes the last element in the Vector. Fails if the Vector is empty." + "Removes the last element in the Vector. Fails if the Vector is empty. + Does not change the capacity of the Vector." input Vector v; protected T null = null; array data = Mutable.access(v.data); Integer sz = Mutable.access(v.size); algorithm - arrayUpdate(data, sz, null); + arrayUpdateNoBoundsChecking(data, sz, null); Mutable.update(v.size, sz - 1); end pop; + function clear + "Removes all elements from the Vector. + Does not change the capacity of the Vector." + input Vector v; + protected + T null = null; + array data = Mutable.access(v.data); + algorithm + for i in 1:Mutable.access(v.size) loop + arrayUpdateNoBoundsChecking(data, i, null); + end for; + + Mutable.update(v.size, 0); + end clear; + function shrink "Removes elements from the Vector until it contains newSize elements, or does nothing if newSize is larger than the size of the Vector. @@ -333,6 +349,17 @@ public arrayUpdateNoBoundsChecking(data, index, value); end update; + function updateNoBounds + "Updates the element at the given one-based index to the given value without + checking if the one-based index is in bounds. This is DANGEROUS and should + only be used when the index is already known to be in bounds." + input Vector v; + input Integer index; + input T value; + algorithm + arrayUpdateNoBoundsChecking(Mutable.access(v.data), index, value); + end updateNoBounds; + function get "Returns the value of the element at the given one-based index. Fails if the index is out of bounds." @@ -350,6 +377,17 @@ public value := arrayGetNoBoundsChecking(data, index); end get; + function getNoBounds + "Returns the value of the element at the given one-based index without + checking if the index is out of bounds. This is DANGEROUS and should only + be used when the index is already known to be in bounds." + input Vector v; + input Integer index; + output T value; + algorithm + value := arrayGetNoBoundsChecking(Mutable.access(v.data), index); + end getNoBounds; + function last "Returns the last element in the array, or fails if the array is empty." input Vector v; @@ -531,6 +569,78 @@ public index := -1; end find; + function all + "Returns true if the given function returns true for all elements in the + Vector, otherwise false." + input Vector v; + input PredFn fn; + output Boolean res; + + partial function PredFn + input T e; + output Boolean res; + end PredFn; + protected + array data = Mutable.access(v.data); + algorithm + for i in 1:Mutable.access(v.size) loop + if not fn(arrayGetNoBoundsChecking(data, i)) then + res := false; + return; + end if; + end for; + + res := true; + end all; + + function any + "Returns true if the given function returns true for any element in the + Vector, otherwise false." + input Vector v; + input PredFn fn; + output Boolean res; + + partial function PredFn + input T e; + output Boolean res; + end PredFn; + protected + array data = Mutable.access(v.data); + algorithm + for i in 1:Mutable.access(v.size) loop + if fn(arrayGetNoBoundsChecking(data, i)) then + res := true; + return; + end if; + end for; + + res := false; + end any; + + function none + "Returns true if the given function returns true for none of the elements in + the Vector, otherwise false." + input Vector v; + input PredFn fn; + output Boolean res; + + partial function PredFn + input T e; + output Boolean res; + end PredFn; + protected + array data = Mutable.access(v.data); + algorithm + for i in 1:Mutable.access(v.size) loop + if fn(arrayGetNoBoundsChecking(data, i)) then + res := false; + return; + end if; + end for; + + res := true; + end none; + function copy "Creates a copy of the given Vector." input Vector v; diff --git a/OMCompiler/Compiler/boot/LoadCompilerSources.mos b/OMCompiler/Compiler/boot/LoadCompilerSources.mos index 493e5e26081..c6c555faee6 100644 --- a/OMCompiler/Compiler/boot/LoadCompilerSources.mos +++ b/OMCompiler/Compiler/boot/LoadCompilerSources.mos @@ -495,6 +495,7 @@ if true then /* Suppress output */ "../Util/SBSet.mo", "../Util/SimulationResults.mo", "../Util/TaskGraphResults.mo", + "../Util/UnorderedMap.mo", "../Util/UnorderedSet.mo", "../Util/Unzip.mo", "../Util/Vector.mo",