Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Faster creation of OrderedDict, revisited #221

Closed
wants to merge 19 commits into from
Closed

Faster creation of OrderedDict, revisited #221

Show file tree
Hide file tree
Changes from 14 commits
Commits
Show all changes
19 commits
Select commit Hold shift + click to select a range
7f9caa1
Trait style dispatch on OrderedDict inner constructor now works. Fall…
Jul 28, 2016
de919ef
Faster creation of ordered dict now only in julia 0.5 and backward co…
Oct 29, 2016
a176da2
one more ordered dict test, trying to get OrderedDict test coverage d…
Nov 14, 2016
a585910
grouped version-specific code better, added at static macro to versio…
Nov 14, 2016
33163b1
newer compat
Nov 16, 2016
bdc4031
inner constructor for one or more Pairs can use the faster constructi…
Nov 17, 2016
9a7f657
do not need the one or more pairs internal constructor
Nov 17, 2016
a8f167b
spacing fixes
Nov 19, 2016
830f402
swapped Pair for => in ordered dict tests
Nov 19, 2016
f6e9769
much simpler way to get ordered dict size information from kmsquire
Nov 19, 2016
9211ff4
some inbounds macros like setindex on ordered dict
Nov 19, 2016
2f9cbb2
ht_keyindex2 instead of hashindex in inner constructor for OrderedDic…
Nov 19, 2016
4f52920
actually make 0.4 work
Nov 20, 2016
2e5aa4b
put at static back
Nov 20, 2016
32a9ddb
a few stray spaces in the OrderedDict type docstring
Nov 20, 2016
49ce624
new construction algo, with tests, that handles duplicated keys
Nov 21, 2016
db96306
another way to resize the OD if duplicated keys appear in the constru…
Nov 21, 2016
a1972c2
some blank lines had indenting spaces
Nov 22, 2016
2678dd4
added test for JuliaLang/julia#15077
Nov 24, 2016
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions REQUIRE
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1 +1,2 @@
julia 0.4
Compat 0.9.4
2 changes: 2 additions & 0 deletions src/DataStructures.jl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,8 @@ __precompile__()

module DataStructures

using Compat

import Base: <, <=, ==, length, isempty, start, next, done,
show, dump, empty!, getindex, setindex!, get, get!,
in, haskey, keys, merge, copy, cat,
Expand Down
91 changes: 55 additions & 36 deletions src/ordered_dict.jl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,8 +8,12 @@ import Base: haskey, get, get!, getkey, delete!, push!, pop!, empty!,
hash, eltype, KeyIterator, ValueIterator, convert, copy,
merge

if VERSION >= v"0.5.0"
import Base: HasLength, HasShape, SizeUnknown, iteratorsize
end

"""
OrderedDict
OrderedDict

`OrderedDict`s are simply dictionaries whose entries have a particular order. The order
refers to insertion order, which allows deterministic iteration over the dictionary or set.
Expand All @@ -21,25 +25,28 @@ type OrderedDict{K,V} <: Associative{K,V}
ndel::Int
dirty::Bool

function OrderedDict()
new(zeros(Int32,16), Array(K,0), Array(V,0), 0, false)
end
OrderedDict() = new(zeros(Int32,16), Array(K,0), Array(V,0), 0, false)

function OrderedDict(kv)
h = OrderedDict{K,V}()
for (k,v) in kv
h[k] = v
end
return h
end
OrderedDict(p::Pair) = setindex!(OrderedDict{K,V}(), p.second, p.first)
function OrderedDict(ps::Pair...)
h = OrderedDict{K,V}()
sizehint!(h, length(ps))
for p in ps
h[p.first] = p.second
(n, slotsz) = get_n_slotsz(kv)
h = new(zeros(Int32,slotsz), Array(K,n), Array(V,n), 0, false)
if n > 0
i = 0
for (k,v) in kv
i = i + 1
index = ht_keyindex2(h, k)
@inbounds h.keys[i] = k
@inbounds h.vals[i] = v
@inbounds h.slots[-index] = i
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Closer, but there needs to be a little more defensive programming here. Since kv is an arbitrary set of key-value pairs, it's possible that a key is repeated, so you'll need to check the return value of index and respond accordingly. You should double check against setindex! to make sure there aren't any other corner cases.

end
else # Unknown length
for (k,v) in kv
h[k] = v
end
end
return h
end

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Minor nitpick: spaces at end of line.

function OrderedDict(d::OrderedDict{K,V})
if d.ndel > 0
rehash!(d)
Expand All @@ -52,24 +59,10 @@ OrderedDict() = OrderedDict{Any,Any}()
OrderedDict(kv::Tuple{}) = OrderedDict()
copy(d::OrderedDict) = OrderedDict(d)


# TODO: this can probably be simplified using `eltype` as a THT (Tim Holy trait)
# OrderedDict{K,V}(kv::Tuple{Vararg{Tuple{K,V}}}) = OrderedDict{K,V}(kv)
# OrderedDict{K }(kv::Tuple{Vararg{Tuple{K,Any}}}) = OrderedDict{K,Any}(kv)
# OrderedDict{V }(kv::Tuple{Vararg{Tuple{Any,V}}}) = OrderedDict{Any,V}(kv)
OrderedDict{K,V}(kv::Tuple{Vararg{Pair{K,V}}}) = OrderedDict{K,V}(kv)
OrderedDict{K}(kv::Tuple{Vararg{Pair{K}}}) = OrderedDict{K,Any}(kv)
OrderedDict{V}(kv::Tuple{Vararg{Pair{TypeVar(:K),V}}}) = OrderedDict{Any,V}(kv)
OrderedDict(kv::Tuple{Vararg{Pair}}) = OrderedDict{Any,Any}(kv)

OrderedDict{K,V}(kv::AbstractArray{Tuple{K,V}}) = OrderedDict{K,V}(kv)
OrderedDict{K,V}(kv::AbstractArray{Pair{K,V}}) = OrderedDict{K,V}(kv)
OrderedDict{K,V}(kv::Associative{K,V}) = OrderedDict{K,V}(kv)

OrderedDict{K,V}(ps::Pair{K,V}...) = OrderedDict{K,V}(ps)
OrderedDict{K}(ps::Pair{K}...,) = OrderedDict{K,Any}(ps)
OrderedDict{V}(ps::Pair{TypeVar(:K),V}...,) = OrderedDict{Any,V}(ps)
OrderedDict(ps::Pair...) = OrderedDict{Any,Any}(ps)

function OrderedDict(kv)
try
Expand All @@ -84,9 +77,35 @@ function OrderedDict(kv)
end
end

dict_with_eltype{K,V}(kv, ::Type{Tuple{K,V}}) = OrderedDict{K,V}(kv)
dict_with_eltype{K,V}(kv, ::Type{Pair{K,V}}) = OrderedDict{K,V}(kv)
dict_with_eltype(kv, t) = OrderedDict{Any,Any}(kv)
@static if VERSION >= v"0.5.0"
get_n_slotsz(kv) = get_n_slotsz(kv, iteratorsize(kv))
get_n_slotsz(kv, isz::SizeUnknown) = (0, 16)
function get_n_slotsz(kv, isz::Union{HasLength, HasShape})
n = length(kv)
slotsz = max(16, (n*3)>>1)
return n, slotsz
end
else
get_n_slotsz(kv) = (0, 16)
end

OrderedDict{K,V}(kv::Tuple{Vararg{Pair{K, V}}}) = OrderedDict{K, V}(kv)
OrderedDict{K}(kv::Tuple{Vararg{Pair{K}}}) = OrderedDict{K, Any}(kv)
OrderedDict{V}(kv::Tuple{Vararg{Pair{TypeVar(:K), V}}}) = OrderedDict{Any, V}(kv)
OrderedDict(kv::Tuple{Vararg{Pair}}) = OrderedDict{Any, Any}(kv)

OrderedDict{K,V}(kv::AbstractArray{Tuple{K, V}}) = OrderedDict{K, V}(kv)
OrderedDict{K,V}(kv::AbstractArray{Pair{K, V}}) = OrderedDict{K, V}(kv)
OrderedDict{K,V}(kv::Associative{K, V}) = OrderedDict{K, V}(kv)

OrderedDict{K,V}(ps::Pair{K, V}...) = OrderedDict{K, V}(ps)
OrderedDict{K}(ps::Pair{K}..., ) = OrderedDict{K, Any}(ps)
OrderedDict{V}(ps::Pair{TypeVar(:K), V}..., ) = OrderedDict{Any, V}(ps)
OrderedDict(ps::Pair...) = OrderedDict{Any, Any}(ps)

dict_with_eltype{K,V}(kv, ::Type{Tuple{K, V}}) = OrderedDict{K, V}(kv)
dict_with_eltype{K,V}(kv, ::Type{Pair{K, V}}) = OrderedDict{K, V}(kv)
dict_with_eltype(kv,t) = OrderedDict{Any, Any}(kv)

similar{K,V}(d::OrderedDict{K,V}) = OrderedDict{K,V}()

Expand Down Expand Up @@ -292,7 +311,7 @@ function setindex!{K,V}(h::OrderedDict{K,V}, v0, key0)
if !isequal(key,key0)
throw(ArgumentError("$key0 is not a valid key for type $K"))
end
v = convert(V, v0)
v = convert(V, v0)

index = ht_keyindex2(h, key)

Expand All @@ -316,7 +335,7 @@ function get!{K,V}(h::OrderedDict{K,V}, key0, default)

index > 0 && return h.vals[index]

v = convert(V, default)
v = convert(V, default)
_setindex!(h, v, key, -index)
return v
end
Expand All @@ -332,7 +351,7 @@ function get!{K,V}(default::Base.Callable, h::OrderedDict{K,V}, key0)
index > 0 && return h.vals[index]

h.dirty = false
v = convert(V, default())
v = convert(V, default())
if h.dirty
index = ht_keyindex2(h, key)
end
Expand Down
10 changes: 6 additions & 4 deletions test/test_ordered_dict.jl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,11 +3,13 @@
@test isa(OrderedDict(), OrderedDict{Any,Any})
@test isa(OrderedDict([(1,2.0)]), OrderedDict{Int,Float64})
@test isa(OrderedDict([("a",1),("b",2)]), OrderedDict{Compat.ASCIIString,Int})
@test isa(OrderedDict(Pair(1, 1.0)), OrderedDict{Int,Float64})
@test isa(OrderedDict(Pair(1, 1.0), Pair(2, 2.0)), OrderedDict{Int,Float64})
@test isa(OrderedDict(Pair(1, 1.0), Pair(2, 2.0), Pair(3, 3.0)), OrderedDict{Int,Float64})
@test isa(OrderedDict(1 => 1.0), OrderedDict{Int,Float64})
@test isa(OrderedDict(1 => 1.0, 2 => 2.0), OrderedDict{Int,Float64})
@test isa(OrderedDict(1 => 1.0, 2 => 2.0, 3 => 3.0), OrderedDict{Int,Float64})
@test isa(OrderedDict([1 => 1.0, 2 => 2.0, 3 => 3.0]), OrderedDict{Int,Float64})

# empty dictionary

d = OrderedDict{Char, Int}()
@test length(d) == 0
@test isempty(d)
Expand All @@ -20,7 +22,7 @@ empty!(d)
@test isempty(d)

# access, modification

d = OrderedDict{Char, Int}()
for c in 'a':'z'
d[c] = c-'a'+1
end
Expand Down