diff --git a/.travis.yml b/.travis.yml index 8e8b4a06a..f33839860 100644 --- a/.travis.yml +++ b/.travis.yml @@ -9,8 +9,8 @@ notifications: email: false script: - if [[ -a .git/shallow ]]; then git fetch --unshallow; fi - - julia -e 'Pkg.clone(pwd()); Pkg.build("HDF5"); Pkg.add("JLDArchives")' - - if [ $TRAVIS_JULIA_VERSION = "nightly" ]; then julia --check-bounds=yes --inline=no --code-coverage=user -e 'include(Pkg.dir("HDF5", "test", "runtests.jl")); include(Pkg.dir("JLDArchives", "test", "runtests.jl"))'; fi + - julia -e 'Pkg.clone(pwd()); Pkg.build("HDF5"); Pkg.add("JLDArchives"); Pkg.checkout("JLDArchives")' + - if [ $TRAVIS_JULIA_VERSION = "nightly" ]; then julia --check-bounds=yes --inline=no --code-coverage=user -e 'reload("HDF5/test/runtests.jl"); workspace(); import LastMain.Compat, LastMain.HDF5; reload("HDF5/src/JLD.jl"); reload("JLDArchives/test/runtests.jl")'; fi - if [ $TRAVIS_JULIA_VERSION = "release" ]; then julia --check-bounds=yes -e 'Pkg.test("HDF5"); Pkg.test("JLDArchives")'; fi after_success: - if [ $TRAVIS_JULIA_VERSION = "nightly" ]; then julia -e 'cd(Pkg.dir("HDF5")); Pkg.add("Coverage"); using Coverage; Coveralls.submit(Coveralls.process_folder())'; fi diff --git a/src/JLD.jl b/src/JLD.jl index 9a629348c..6a95c1ade 100644 --- a/src/JLD.jl +++ b/src/JLD.jl @@ -9,15 +9,6 @@ import HDF5: close, dump, exists, file, getindex, setindex!, g_create, g_open, o HDF5ReferenceObj, HDF5BitsKind, ismmappable, readmmap import Base: length, endof, show, done, next, ndims, start, delete!, size, sizeof -# .jld files written before v"0.4.0-dev+1419" might have UInt32 instead of UInt32 as the typename string. -# See julia issue #8907 -if VERSION >= v"0.4.0-dev+1419" - julia_type(s::AbstractString) = _julia_type(replace(s, r"UInt(?=\d{1,3})", "UInt")) -else - julia_type(s::AbstractString) = _julia_type(s) -end - - const magic_base = "Julia data file (HDF5), version " const version_current = v"0.1" const pathrefs = "/_refs" @@ -111,6 +102,11 @@ immutable AssociativeWrapper{K,V,T<:Associative} values::Vector{V} end +# Wrapper for SimpleVector +immutable SimpleVectorWrapper + elements::Vector +end + include("jld_types.jl") file(x::JldFile) = x @@ -294,7 +290,7 @@ function delete!(parent::Union(JldFile, JldGroup), path::ByteString) exists(parent, path) || error("$path does not exist in $parent") delete!(parent[path]) end -delete!(parent::Union(JldFile, JldGroup), args::(ByteString...)) = for a in args delete!(parent,a) end +delete!(parent::Union(JldFile, JldGroup), args::@compat Tuple{Vararg{ByteString}}) = for a in args delete!(parent,a) end ismmappable(obj::JldDataset) = ismmappable(obj.plain) readmmap(obj::JldDataset, args...) = readmmap(obj.plain, args...) setindex!(parent::Union(JldFile, JldGroup), val, path::ASCIIString) = write(parent, path, val) @@ -376,11 +372,16 @@ function after_read{K,V,T}(x::AssociativeWrapper{K,V,T}) ret end +# Special case for SimpleVector +if VERSION >= v"0.4.0-dev+4319" + after_read(x::SimpleVectorWrapper) = Base.svec(x.elements...) +end + ## Arrays # Read an array function read_array(obj::JldDataset, dtype::HDF5Datatype, dspace_id::HDF5.Hid, dsel_id::HDF5.Hid, - dims::(Int...)=convert((Int...), HDF5.h5s_get_simple_extent_dims(dspace_id)[1])) + dims::(@compat Tuple{Vararg{Int}})=convert((@compat Tuple{Vararg{Int}}), HDF5.h5s_get_simple_extent_dims(dspace_id)[1])) if HDF5.h5t_get_class(dtype) == HDF5.H5T_REFERENCE return read_refs(obj, refarray_eltype(obj), dspace_id, dsel_id, dims) else @@ -390,7 +391,7 @@ end # Arrays of basic HDF5 kinds function read_vals{S<:HDF5BitsKind}(obj::JldDataset, dtype::HDF5Datatype, T::Union(Type{S}, Type{Complex{S}}), - dspace_id::HDF5.Hid, dsel_id::HDF5.Hid, dims::(Int...)) + dspace_id::HDF5.Hid, dsel_id::HDF5.Hid, dims::@compat Tuple{Vararg{Int}}) if obj.file.mmaparrays && HDF5.iscontiguous(obj.plain) && dsel_id == HDF5.H5S_ALL readmmap(obj.plain, Array{T}) else @@ -402,7 +403,7 @@ end # Arrays of immutables/bitstypes function read_vals(obj::JldDataset, dtype::HDF5Datatype, T::Type, dspace_id::HDF5.Hid, - dsel_id::HDF5.Hid, dims::(Int...)) + dsel_id::HDF5.Hid, dims::@compat Tuple{Vararg{Int}}) out = Array(T, dims) # Empty objects don't need to be read at all T.size == 0 && !T.mutable && return out @@ -437,7 +438,7 @@ end # Arrays of references function read_refs{T}(obj::JldDataset, ::Type{T}, dspace_id::HDF5.Hid, dsel_id::HDF5.Hid, - dims::(Int...)) + dims::@compat Tuple{Vararg{Int}}) refs = Array(HDF5ReferenceObj, dims) HDF5.h5d_read(obj.plain.id, HDF5.H5T_STD_REF_OBJ, dspace_id, dsel_id, HDF5.H5P_DEFAULT, refs) @@ -488,7 +489,7 @@ write(parent::Union(JldFile, JldGroup), name::ByteString, close(_write(parent, name, data, wsession; kargs...)) # Pick whether to use compact or default storage based on data size -function dset_create_properties(parent, sz::Int, obj, chunk=Int[]; mmap = false) +function dset_create_properties(parent, sz::Int, obj, chunk=Int[]; mmap::Bool=false) if sz <= 8192 && !ismmapped(parent) && !mmap return compact_properties(), false end @@ -634,10 +635,9 @@ write_ref(parent::JldGroup, data, wsession::JldWriteSession) = write_ref(file(parent), data, wsession) # Special case for associative, to rehash keys -function _write(parent::Union(JldFile, JldGroup), name::ByteString, - d::Associative, wsession::JldWriteSession; kargs...) +function _write{K,V}(parent::Union(JldFile, JldGroup), name::ByteString, + d::Associative{K,V}, wsession::JldWriteSession; kargs...) n = length(d) - K, V = eltype(d) ks = Array(K, n) vs = Array(V, n) i = 0 @@ -645,7 +645,14 @@ function _write(parent::Union(JldFile, JldGroup), name::ByteString, ks[i+=1] = k vs[i] = v end - write_compound(parent, name, AssociativeWrapper{K,V,typeof(d)}(ks, vs), wsession) + write_compound(parent, name, AssociativeWrapper{K,V,typeof(d)}(ks, vs), wsession; kargs...) +end + +# Special case for SimpleVector +if VERSION >= v"0.4.0-dev+4319" + _write(parent::Union(JldFile, JldGroup), name::ByteString, + d::SimpleVector, wsession::JldWriteSession; kargs...) = + write_compound(parent, name, SimpleVectorWrapper([d...]), wsession; kargs...) end # Expressions, drop line numbers @@ -757,14 +764,66 @@ is_valid_type_ex{T}(::T) = isbits(T) is_valid_type_ex(e::Expr) = ((e.head == :curly || e.head == :tuple || e.head == :.) && all(map(is_valid_type_ex, e.args))) || (e.head == :call && (e.args[1] == :Union || e.args[1] == :TypeVar)) -# Work around https://github.com/JuliaLang/julia/issues/8226 +if VERSION >= v"0.4.0-dev+1419" + const typemap_Core = @compat Dict( + :Uint8 => :UInt8, + :Uint16 => :Uint16, + :Uint32 => :UInt32, + :Uint64 => :UInt64, + :Nothing => :Void + ) +else + const typemap_Core = @compat Dict( + :UInt8 => :Uint8, + :UInt16 => :Uint16, + :UInt32 => :Uint32, + :UInt64 => :Uint64, + :Void => :Nothing + ) +end + const _typedict = Dict{UTF8String,Type}() -_typedict["Core.Type{TypeVar(:T,Union(Core.Any,Core.Undef))}"] = Type -function _julia_type(s::AbstractString) +fixtypes(typ) = typ +@eval begin + function fixtypes(typ::Expr) + if typ.head == :. + if length(typ.args) == 2 && typ.args[1] == :Core + arg = typ.args[2].value + return Expr(:., :Core, QuoteNode(get(typemap_Core, arg, arg))) + else + return typ + end + elseif typ == :(Core.Type{TypeVar(:T,Union(Core.Any,Core.Undef))}) || typ == :(Core.Type{TypeVar(:T)}) + # Work around https://github.com/JuliaLang/julia/issues/8226 and the removal of Top + return :(Core.Type) + end + + for i = 1:length(typ.args) + typ.args[i] = fixtypes(typ.args[i]) + end + + $(if VERSION >= v"0.4.0-dev+4319" + quote + if typ.head == :tuple + return Expr(:curly, :Tuple, typ.args...) + end + end + else + quote + if typ.head == :curly && !isempty(typ.args) && typ.args[1] == :(Core.Tuple) + return Expr(:tuple, typ.args[2:end]...) + end + end + end) + typ + end +end + +function julia_type(s::AbstractString) typ = get(_typedict, s, UnconvertedType) if typ == UnconvertedType - typ = julia_type(parse(s)) + typ = julia_type(fixtypes(parse(s))) if typ != UnsupportedType _typedict[s] = typ end @@ -811,13 +870,15 @@ function full_typename(io::IO, file::JldFile, tv::TypeVar) print(io, ')') end end -function full_typename(io::IO, file::JldFile, jltype::(Type...)) - print(io, '(') - for t in jltype - full_typename(io, file, t) - print(io, ',') +if VERSION < v"0.4.0-dev+4319" + function full_typename(io::IO, file::JldFile, jltype::@compat Tuple{Vararg{Type}}) + print(io, '(') + for t in jltype + full_typename(io, file, t) + print(io, ',') + end + print(io, ')') end - print(io, ')') end function full_typename(io::IO, ::JldFile, x) # Only allow bitstypes that show as AST literals and make sure that they @@ -826,7 +887,7 @@ function full_typename(io::IO, ::JldFile, x) # A different implementation will be required to support custom immutables # or things as simple as Int16(1). s = sprint(show, x) - if isbits(x) && parse(s) === x + if isbits(x) && parse(s) === x && !isa(x, Tuple) print(io, s) else error("type parameters with objects of type ", typeof(x), " are currently unsupported") @@ -859,6 +920,8 @@ function full_typename(io::IO, file::JldFile, jltype::DataType) full_typename(io, file, jltype.parameters[i]) end print(io, '}') + elseif jltype <: Tuple + print(io, "{}") end end function full_typename(file::JldFile, x) @@ -976,7 +1039,7 @@ function save(filename::AbstractString, dict::Associative; compress::Bool=false) end # Or the names and values may be specified as alternating pairs function save(filename::AbstractString, name::AbstractString, value, pairs...; compress::Bool=false) - if isodd(length(pairs)) || !isa(pairs[1:2:end], (AbstractString...)) + if isodd(length(pairs)) || !isa(pairs[1:2:end], @compat Tuple{Vararg{AbstractString}}) throw(ArgumentError("arguments must be in name-value pairs")) end jldopen(filename, "w"; compress=compress) do file @@ -1001,7 +1064,7 @@ function load(filename::AbstractString, varname::AbstractString) end end load(filename::AbstractString, varnames::AbstractString...) = load(filename, varnames) -function load(filename::AbstractString, varnames::(AbstractString...)) +function load(filename::AbstractString, varnames::@compat Tuple{Vararg{AbstractString}}) jldopen(filename, "r") do file map((var)->read(file, var), varnames) end diff --git a/src/JLD00.jl b/src/JLD00.jl index fe3a101cb..77cbdfe44 100644 --- a/src/JLD00.jl +++ b/src/JLD00.jl @@ -8,6 +8,7 @@ using HDF5, Compat import HDF5: close, dump, exists, file, getindex, setindex!, g_create, g_open, o_delete, name, names, read, size, write, HDF5ReferenceObj, HDF5BitsKind, ismmappable, readmmap import Base: length, endof, show, done, next, start, delete! +import JLD if !isdefined(:setfield!) const setfield! = setfield @@ -226,7 +227,7 @@ function delete!(parent::Union(JldFile, JldGroup), path::ByteString) exists(parent, path) || error("$path does not exist in $parent") delete!(parent[path]) end -delete!(parent::Union(JldFile, JldGroup), args::(ByteString...)) = for a in args delete!(parent,a) end +delete!(parent::Union(JldFile, JldGroup), args::@compat Tuple{Vararg{ByteString}}) = for a in args delete!(parent,a) end ismmappable(obj::JldDataset) = ismmappable(obj.plain) readmmap(obj::JldDataset, args...) = readmmap(obj.plain, args...) setindex!(parent::Union(JldFile, JldGroup), val, path::ASCIIString) = write(parent, path, val) @@ -367,7 +368,7 @@ end # Nothing read(obj::JldDataset, ::Type{Nothing}) = nothing -read(obj::JldDataset, ::Type{Bool}) = bool(read(obj, UInt8)) +read(obj::JldDataset, ::Type{Bool}) = read(obj, UInt8) != 0 # Types read{T}(obj::JldDataset, ::Type{Type{T}}) = T @@ -397,7 +398,7 @@ read(obj::JldDataset, ::Type{Symbol}) = symbol(read(obj.plain, ByteString)) read{N}(obj::JldDataset, ::Type{Array{Symbol,N}}) = map(symbol, read(obj.plain, Array{ByteString})) # Char -read(obj::JldDataset, ::Type{Char}) = char(read(obj.plain, UInt32)) +read(obj::JldDataset, ::Type{Char}) = @compat Char(read(obj.plain, UInt32)) # UTF16String (not defined in julia 0.2) if VERSION >= v"0.3-" @@ -436,7 +437,7 @@ end # CompositeKind function read(obj::JldDataset, T::DataType) - if isempty(T.names) && T.size > 0 + if isempty(fieldnames(T)) && T.size > 0 return read_bitstype(obj, T) end local x @@ -453,12 +454,12 @@ function read(obj::JldDataset, T::DataType) if length(v) == 0 x = ccall(:jl_new_struct, Any, (Any,Any...), T) else - n = T.names + n = fieldnames(T) if length(v) != length(n) error("Wrong number of fields") end if !T.mutable - x = ccall(:jl_new_structv, Any, (Any,Ptr{Void},UInt32), T, v, length(T.names)) + x = ccall(:jl_new_structv, Any, (Any,Ptr{Void},UInt32), T, v, length(fieldnames(T))) else x = ccall(:jl_new_struct_uninit, Any, (Any,), T) for i = 1:length(v) @@ -583,9 +584,9 @@ write(parent::Union(JldFile, JldGroup), name::ByteString, n::Nothing) = write(pa # Types # the first is needed to avoid an ambiguity warning if isdefined(Core, :Top) - write{T<:Top}(parent::Union(JldFile, JldGroup), name::ByteString, t::(Type{T}...)) = write(parent, name, Any[t...], "Tuple") + write{T<:Top}(parent::Union(JldFile, JldGroup), name::ByteString, t::@compat Tuple{Vararg{Type{T}}}) = write(parent, name, Any[t...], "Tuple") else - write{T}(parent::Union(JldFile, JldGroup), name::ByteString, t::(Type{T}...)) = write(parent, name, Any[t...], "Tuple") + write{T}(parent::Union(JldFile, JldGroup), name::ByteString, t::@compat Tuple{Vararg{Type{T}}}) = write(parent, name, Any[t...], "Tuple") end write{T}(parent::Union(JldFile, JldGroup), name::ByteString, t::Type{T}) = write(parent, name, nothing, string("Type{", full_typename(t), "}")) @@ -720,7 +721,7 @@ write(parent::Union(JldFile, JldGroup), name::ByteString, s; rootmodule="") = wr function write_composite(parent::Union(JldFile, JldGroup), name::ByteString, s; rootmodule="") T = typeof(s) - if isempty(T.names) + if isempty(fieldnames(T)) if T.size > 0 return write_bitstype(parent, name, s) end @@ -730,7 +731,7 @@ function write_composite(parent::Union(JldFile, JldGroup), name::ByteString, s; return end Tname = string(T.name.name) - n = T.names + n = fieldnames(T) local gtypes if !exists(file(parent), pathtypes) gtypes = g_create(file(parent), pathtypes) @@ -803,7 +804,7 @@ function has_pointer_field(obj::Tuple, name) end function has_pointer_field(obj, name) - names = typeof(obj).names + names = fieldnames(typeof(obj)) for fieldname in names if isdefined(obj, fieldname) x = getfield(obj, fieldname) @@ -937,6 +938,7 @@ function _julia_type(s::AbstractString) typ = get(_typedict, s, UnconvertedType) if typ == UnconvertedType e = parse(s) + e = JLD.fixtypes(e) typ = UnsupportedType if is_valid_type_ex(e) try # try needed to catch undefined symbols @@ -975,8 +977,9 @@ function full_typename(tv::TypeVar) "TypeVar(:$(tv.name),$(full_typename(tv.lb)),$(full_typename(tv.ub)))" end end -full_typename(jltype::(Type...)) = length(jltype) == 1 ? @sprintf("(%s,)", full_typename(jltype[1])) : - @sprintf("(%s)", join(map(full_typename, jltype), ",")) +full_typename(jltype::@compat Tuple{Vararg{Type}}) = + length(jltype) == 1 ? @sprintf("(%s,)", full_typename(jltype[1])) : + @sprintf("(%s)", join(map(full_typename, jltype), ",")) full_typename(x) = string(x) function full_typename(jltype::DataType) #tname = "$(jltype.name.module).$(jltype.name)" @@ -1117,7 +1120,7 @@ function load(filename::AbstractString, varname::AbstractString) end end load(filename::AbstractString, varnames::AbstractString...) = load(filename, varnames) -function load(filename::AbstractString, varnames::(AbstractString...)) +function load(filename::AbstractString, varnames::@compat Tuple{Vararg{AbstractString}}) jldopen(filename, "r") do file map((var)->read(file, var), varnames) end diff --git a/src/jld_types.jl b/src/jld_types.jl index f203a86b0..1bf31a5aa 100644 --- a/src/jld_types.jl +++ b/src/jld_types.jl @@ -10,6 +10,20 @@ const BUILTIN_TYPES = Set([Symbol, Type, UTF16String, BigFloat, BigInt]) const H5CONVERT_DEFINED = ObjectIdDict() const JLCONVERT_DEFINED = ObjectIdDict() +if VERSION >= v"0.4.0-dev+4319" + const EMPTY_TUPLE_TYPE = Tuple{} + typealias TypesType SimpleVector + typealias TupleType{T<:Tuple} Type{T} + tupletypes(T::TupleType) = T.parameters + typetuple(types) = Tuple{types...} +else + const EMPTY_TUPLE_TYPE = () + typealias TypesType (Type...) + typealias TupleType (Type...) + tupletypes(T::TupleType) = T + typetuple(types) = tuple(types...) +end + ## Helper functions # Holds information about the mapping between a Julia and HDF5 type @@ -20,7 +34,7 @@ immutable JldTypeInfo end # Get information about the HDF5 types corresponding to Julia types -function JldTypeInfo(parent::JldFile, types::(Type...), commit::Bool) +function JldTypeInfo(parent::JldFile, types::TypesType, commit::Bool) dtypes = Array(JldDatatype, length(types)) offsets = Array(Int, length(types)) offset = 0 @@ -257,7 +271,7 @@ function h5type{T<:Type}(parent::JldFile, ::Type{T}, commit::Bool) id = HDF5.h5t_create(HDF5.H5T_COMPOUND, 8) HDF5.h5t_insert(id, "typename_", 0, h5fieldtype(parent, UTF8String, commit)) dtype = HDF5Datatype(id, parent.plain) - commit ? commit_datatype(parent, dtype, Type) : JldDatatype(dtype, -1) + out = commit ? commit_datatype(parent, dtype, Type) : JldDatatype(dtype, -1) end gen_h5convert{T<:Type}(::JldFile, ::Type{T}) = nothing @@ -292,16 +306,13 @@ h5fieldtype{T,N}(parent::JldFile, ::Type{Array{T,N}}, ::Bool) = JLD_REF_TYPE ## Tuples if INLINE_TUPLE - h5fieldtype(parent::JldFile, T::(Type...), commit::Bool) = + h5fieldtype(parent::JldFile, T::TupleType, commit::Bool) = isleaftype(T) ? h5type(parent, T, commit) : JLD_REF_TYPE else - h5fieldtype(parent::JldFile, T::(Type...), ::Bool) = JLD_REF_TYPE + h5fieldtype(parent::JldFile, T::TupleType, ::Bool) = JLD_REF_TYPE end -function h5type(parent::JldFile, T::(Type...), commit::Bool) - !isa(T, (Union(Tuple, DataType)...)) && unknown_type_err(T) - T = T::(Union(Tuple, DataType)...) - +function h5type(parent::JldFile, T::TupleType, commit::Bool) haskey(parent.jlh5type, T) && return parent.jlh5type[T] # Tuples should always be concretely typed, unless we're # reconstructing a tuple, in which case commit will be false @@ -321,7 +332,7 @@ function h5type(parent::JldFile, T::(Type...), commit::Bool) dtype = HDF5Datatype(id, parent.plain) if commit jlddtype = commit_datatype(parent, dtype, T) - if isempty(T) + if T == EMPTY_TUPLE_TYPE # to allow recovery of empty tuples, which HDF5 does not allow a_write(dtype, "empty", @compat UInt8(1)) end @@ -331,13 +342,14 @@ function h5type(parent::JldFile, T::(Type...), commit::Bool) end end -function gen_jlconvert(typeinfo::JldTypeInfo, T::(Type...)) +function gen_jlconvert(typeinfo::JldTypeInfo, T::TupleType) haskey(JLCONVERT_DEFINED, T) && return ex = Expr(:block) args = ex.args tup = Expr(:tuple) tupargs = tup.args + types = tupletypes(T) for i = 1:length(typeinfo.dtypes) h5offset = typeinfo.offsets[i] field = symbol(string("field", i)) @@ -345,7 +357,7 @@ function gen_jlconvert(typeinfo::JldTypeInfo, T::(Type...)) if HDF5.h5t_get_class(typeinfo.dtypes[i]) == HDF5.H5T_REFERENCE push!(args, :($field = read_ref(file, unsafe_load(convert(Ptr{HDF5ReferenceObj}, ptr)+$h5offset)))) else - push!(args, :($field = jlconvert($(T[i]), file, ptr+$h5offset))) + push!(args, :($field = jlconvert($(types[i]), file, ptr+$h5offset))) end push!(tupargs, field) end @@ -382,7 +394,7 @@ function h5type(parent::JldFile, T::ANY, commit::Bool) id = HDF5.h5t_create(HDF5.H5T_COMPOUND, typeinfo.size) for i = 1:length(typeinfo.offsets) fielddtype = typeinfo.dtypes[i] - HDF5.h5t_insert(id, mangle_name(fielddtype, T.names[i]), typeinfo.offsets[i], fielddtype) + HDF5.h5t_insert(id, mangle_name(fielddtype, fieldnames(T)[i]), typeinfo.offsets[i], fielddtype) end end @@ -410,11 +422,11 @@ function _gen_jlconvert_type(typeinfo::JldTypeInfo, T::ANY) push!(args, quote ref = unsafe_load(convert(Ptr{HDF5ReferenceObj}, ptr)+$h5offset) if ref != HDF5.HDF5ReferenceObj_NULL - out.$(T.names[i]) = convert($(T.types[i]), read_ref(file, ref)) + out.$(fieldnames(T)[i]) = convert($(T.types[i]), read_ref(file, ref)) end end) else - push!(args, :(out.$(T.names[i]) = jlconvert($(T.types[i]), file, ptr+$h5offset))) + push!(args, :(out.$(fieldnames(T)[i]) = jlconvert($(T.types[i]), file, ptr+$h5offset))) end end @eval function jlconvert(::Type{$T}, file::JldFile, ptr::Ptr) @@ -434,7 +446,20 @@ function _gen_jlconvert_immutable(typeinfo::JldTypeInfo, T::ANY) h5offset = typeinfo.offsets[i] jloffset = jloffsets[i] - if HDF5.h5t_get_class(typeinfo.dtypes[i]) == HDF5.H5T_REFERENCE + if isa(T.types[i], TupleType) && VERSION >= v"0.4.0-dev+4319" && T.types[i].pointerfree + # We continue to store tuples as references for the sake of + # backwards compatibility, but on 0.4 they are now stored + # inline + push!(args, quote + ref = unsafe_load(convert(Ptr{HDF5ReferenceObj}, ptr)+$h5offset) + if ref == HDF5.HDF5ReferenceObj_NULL + warn("""A pointerfree tuple field was undefined. + This is not supported in Julia 0.4 and the corresponding tuple will be uninitialized.""") + else + unsafe_store!(convert(Ptr{$(T.types[i])}, out)+$jloffset, read_ref(file, ref)) + end + end) + elseif HDF5.h5t_get_class(typeinfo.dtypes[i]) == HDF5.H5T_REFERENCE obj = gensym("obj") push!(args, quote ref = unsafe_load(convert(Ptr{HDF5ReferenceObj}, ptr)+$h5offset) @@ -491,7 +516,7 @@ const DONT_STORE_SINGLETON_IMMUTABLES = VERSION >= v"0.4.0-dev+385" function gen_jlconvert(typeinfo::JldTypeInfo, T::ANY) haskey(JLCONVERT_DEFINED, T) && return - if isempty(T.names) + if isempty(fieldnames(T)) if T.size == 0 @eval begin jlconvert(::Type{$T}, ::JldFile, ::Ptr) = $T() @@ -523,11 +548,12 @@ end ## Common functions for all non-special types (including gen_h5convert) # Whether this datatype should be stored as opaque -isopaque(t::(Type...)) = isa(t, ()) -isopaque(t::DataType) = isempty(t.names) +isopaque(t::TupleType) = t == EMPTY_TUPLE_TYPE +# isopaque(t::DataType) = isempty(fieldnames(t)) +isopaque(t::DataType) = isa(t, TupleType) ? t == EMPTY_TUPLE_TYPE : isempty(fieldnames(t)) # The size of this datatype in the HDF5 file (if opaque) -opaquesize(t::(DataType...)) = 1 +opaquesize(t::TupleType) = 1 opaquesize(t::DataType) = max(1, t.size) # Whether a type that is stored inline in HDF5 should be stored as a @@ -535,7 +561,7 @@ opaquesize(t::DataType) = max(1, t.size) # true for some unions of special types defined above, unless either # INLINE_TUPLE or INLINE_POINTER_IMMUTABLE is true. uses_reference(T::DataType) = !T.pointerfree -uses_reference(::Tuple) = true +uses_reference(::TupleType) = true uses_reference(::UnionType) = true unknown_type_err(T) = @@ -548,19 +574,21 @@ gen_h5convert(parent::JldFile, T) = # There is no point in specializing this function _gen_h5convert(parent::JldFile, T::ANY) dtype = parent.jlh5type[T].dtype - istuple = isa(T, Tuple) + istuple = isa(T, TupleType) + + if isopaque(T) + if T.size == 0 + @eval h5convert!(out::Ptr, ::JldFile, x::$T, ::JldWriteSession) = nothing + else + @eval h5convert!(out::Ptr, ::JldFile, x::$T, ::JldWriteSession) = + unsafe_store!(convert(Ptr{$T}, out), x) + end + return + end + if istuple - types = T + types = tupletypes(T) else - if isopaque(T::DataType) - if (T::DataType).size == 0 - @eval h5convert!(out::Ptr, ::JldFile, x::$T, ::JldWriteSession) = nothing - else - @eval h5convert!(out::Ptr, ::JldFile, x::$T, ::JldWriteSession) = - unsafe_store!(convert(Ptr{$T}, out), x) - end - return - end types = (T::DataType).types end @@ -704,9 +732,9 @@ function reconstruct_type(parent::JldFile, dtype::HDF5Datatype, savedname::Abstr end end - if startswith(savedname, "(") + if startswith(savedname, "(") || startswith(savedname, "Core.Tuple{") # We're reconstructing a tuple - tuple(fieldtypes...) + typetuple(fieldtypes) else # We're reconstructing some other type @eval begin diff --git a/src/plain.jl b/src/plain.jl index 8b4e605f4..3c7e6179c 100644 --- a/src/plain.jl +++ b/src/plain.jl @@ -431,10 +431,8 @@ end type EmptyArray{T}; end # Stub types to encode fixed-size arrays for H5T_ARRAY -immutable DimSize{N}; end # Int-wrapper (can't use tuple of Int as param) -immutable FixedArray{T,D<:(DimSize...)}; end -dimsize{N}(::Type{DimSize{N}}) = N -size{T,D}(::Type{FixedArray{T,D}}) = map(dimsize, D)::(Int...) +immutable FixedArray{T,D}; end +size{T,D}(::Type{FixedArray{T,D}}) = D eltype{T,D}(::Type{FixedArray{T,D}}) = T # VLEN objects @@ -593,7 +591,7 @@ function h5read(filename, name::ByteString) dat end -function h5read(filename, name::ByteString, indices::(Union(Range{Int},Int,Colon)...)) +function h5read(filename, name::ByteString, indices::@compat Tuple{Vararg{Union(Range{Int},Int,Colon)}}) local dat fid = h5open(filename, "r") try @@ -767,7 +765,7 @@ function d_create(parent::Union(HDF5File, HDF5Group), path::ByteString, dtype::H HDF5Dataset(h5d_create(parent, path, dtype.id, dspace.id, _link_properties(path), p.id, H5P_DEFAULT), file(parent)) end d_create(parent::Union(HDF5File, HDF5Group), path::ByteString, dtype::HDF5Datatype, dspace_dims::Dims, prop1::ASCIIString, val1, pv...) = d_create(checkvalid(parent), path, dtype, dataspace(dspace_dims), prop1, val1, pv...) -d_create(parent::Union(HDF5File, HDF5Group), path::ByteString, dtype::HDF5Datatype, dspace_dims::(Dims,Dims), prop1::ASCIIString, val1, pv...) = d_create(checkvalid(parent), path, dtype, dataspace(dspace_dims[1], max_dims=dspace_dims[2]), prop1, val1, pv...) +d_create(parent::Union(HDF5File, HDF5Group), path::ByteString, dtype::HDF5Datatype, dspace_dims::(@compat Tuple{Dims,Dims}), prop1::ASCIIString, val1, pv...) = d_create(checkvalid(parent), path, dtype, dataspace(dspace_dims[1], max_dims=dspace_dims[2]), prop1, val1, pv...) d_create(parent::Union(HDF5File, HDF5Group), path::ByteString, dtype::Type, dspace_dims, prop1::ASCIIString, val1, pv...) = d_create(checkvalid(parent), path, datatype(dtype), dataspace(dspace_dims[1], max_dims=dspace_dims[2]), prop1, val1, pv...) # Note that H5Tcreate is very different; H5Tcommit is the analog of these others @@ -889,7 +887,7 @@ function size(obj::Union(HDF5Dataset, HDF5Attribute)) dspace = dataspace(obj) dims, maxdims = get_dims(dspace) close(dspace) - convert((Int...), dims) + convert((@compat Tuple{Vararg{Int}}), dims) end size(dset::Union(HDF5Dataset, HDF5Attribute), d) = d > ndims(dset) ? 1 : size(dset)[d] length(dset::Union(HDF5Dataset, HDF5Attribute)) = prod(size(dset)) @@ -1054,7 +1052,7 @@ dataspace(attr::HDF5Attribute) = HDF5Dataspace(h5a_get_space(checkvalid(attr).id # Create a dataspace from in-memory types dataspace{T<:HDF5BitsKind}(x::T) = HDF5Dataspace(h5s_create(H5S_SCALAR)) -function _dataspace(sz::(Int...), max_dims::Union(Dims, ())=()) +function _dataspace(sz::(@compat Tuple{Vararg{Int}}), max_dims::Union(Dims, @compat Tuple{})=()) dims = Array(Hsize, length(sz)) any_zero = false for i = 1:length(sz) @@ -1078,13 +1076,13 @@ function _dataspace(sz::(Int...), max_dims::Union(Dims, ())=()) end HDF5Dataspace(space_id) end -dataspace(A::Array; max_dims::Union(Dims, ()) = ()) = _dataspace(size(A), max_dims) +dataspace(A::Array; max_dims::Union(Dims, @compat Tuple{}) = ()) = _dataspace(size(A), max_dims) dataspace(str::ByteString) = HDF5Dataspace(h5s_create(H5S_SCALAR)) -dataspace(R::Array{HDF5ReferenceObj}; max_dims::Union(Dims, ())=()) = _dataspace(size(R), max_dims) -dataspace(v::HDF5Vlen; max_dims::Union(Dims, ())=()) = _dataspace(size(v.data), max_dims) +dataspace(R::Array{HDF5ReferenceObj}; max_dims::Union(Dims, @compat Tuple{})=()) = _dataspace(size(R), max_dims) +dataspace(v::HDF5Vlen; max_dims::Union(Dims, @compat Tuple{})=()) = _dataspace(size(v.data), max_dims) dataspace(n::Nothing) = HDF5Dataspace(h5s_create(H5S_NULL)) -dataspace(sz::Dims; max_dims::Union(Dims, ())=()) = _dataspace(sz, max_dims) -dataspace(sz1::Int, sz2::Int, sz3::Int...; max_dims::Union(Dims, ())=()) = _dataspace(tuple(sz1, sz2, sz3...), max_dims) +dataspace(sz::Dims; max_dims::Union(Dims, @compat Tuple{})=()) = _dataspace(sz, max_dims) +dataspace(sz1::Int, sz2::Int, sz3::Int...; max_dims::Union(Dims, @compat Tuple{})=()) = _dataspace(tuple(sz1, sz2, sz3...), max_dims) # Get the array dimensions from a dataspace or a dataset # Returns both dims and maxdims @@ -1578,9 +1576,7 @@ function _setindex!(dset::HDF5Dataset,T::Type, X::Array, indices::Union(Range{In if !(T.parameters[1]<:HDF5BitsKind) error("Dataset indexing (hyperslab) is available only for bits types") end - if length(X) != prod([length(idxs) for idxs in - filter(idx -> isa(idx, Ranges), - [indices[i] for i in 1:length(indices)])]) + if length(X) != prod(map(length, indices)) error("number of elements in range and length of array must be equal") end if eltype(X) != T.parameters[1] @@ -1835,7 +1831,7 @@ end ### Utilities for generating ccall wrapper functions programmatically ### -function ccallexpr(lib::AbstractString, ccallsym::Symbol, outtype, argtypes::Tuple, argsyms::Tuple) +function ccallexpr(lib::AbstractString, ccallsym::Symbol, outtype, argtypes::(@compat Tuple), argsyms::@compat Tuple) ccallargs = Any[Expr(:tuple, Expr(:quote, ccallsym), lib), outtype, Expr(:tuple, argtypes...)] ccallargs = ccallsyms(ccallargs, length(argtypes), argsyms) Expr(:ccall, ccallargs...) @@ -2135,7 +2131,7 @@ function hdf5array(objtype) h5t_get_array_dims(objtype.id, dims) eltyp = HDF5Datatype(h5t_get_super(objtype.id)) T = hdf5_to_julia_eltype(eltyp) - dimsizes = ntuple(nd, i->DimSize{@compat Int(dims[nd-i+1])}) # reverse order + dimsizes = ntuple(nd, i->@compat Int(dims[nd-i+1])) # reverse order FixedArray{T, dimsizes} end diff --git a/test/extend_test.jl b/test/extend_test.jl index 42ea39709..0197b2adf 100644 --- a/test/extend_test.jl +++ b/test/extend_test.jl @@ -1,4 +1,4 @@ -using HDF5 +using HDF5, Compat fn = joinpath(tempdir(),"test.h5") diff --git a/test/jld.jl b/test/jld.jl index e896362ff..c1611a002 100644 --- a/test/jld.jl +++ b/test/jld.jl @@ -77,7 +77,7 @@ sa_utf8 = [:α, :β] # SubArray (to test tuple type params) subarray = sub([1:5;], 1:5) # Array of empty tuples (to test tuple type params) -arr_empty_tuple = ()[] +arr_empty_tuple = (@compat Tuple{})[] immutable EmptyImmutable end emptyimmutable = EmptyImmutable() arr_emptyimmutable = [emptyimmutable] @@ -205,13 +205,19 @@ bitsparamint16 = BitsParams{@compat Int16(1)}() # Tuple of tuples tuple_of_tuples = (1, 2, (3, 4, [5, 6]), [7, 8]) +# SimpleVector +if VERSION >= v"0.4.0-dev+4319" + simplevec = Base.svec(1, 2, Int64, "foo") + iseq(x::SimpleVector, y::SimpleVector) = collect(x) == collect(y) +end + iseq(x,y) = isequal(x,y) iseq(x::MyStruct, y::MyStruct) = (x.len == y.len && x.data == y.data) iseq(x::MyImmutable, y::MyImmutable) = (isequal(x.x, y.x) && isequal(x.y, y.y) && isequal(x.z, y.z)) iseq(x::Union(EmptyTI, EmptyTT), y::Union(EmptyTI, EmptyTT)) = isequal(x.x, y.x) iseq(c1::Array{Base.Sys.CPUinfo}, c2::Array{Base.Sys.CPUinfo}) = length(c1) == length(c2) && all([iseq(c1[i], c2[i]) for i = 1:length(c1)]) function iseq(c1::Base.Sys.CPUinfo, c2::Base.Sys.CPUinfo) - for n in Base.Sys.CPUinfo.names + for n in fieldnames(Base.Sys.CPUinfo) if getfield(c1, n) != getfield(c2, n) return false end @@ -295,8 +301,7 @@ rm(fn) for compress in (true,false) - fnc = compress ? fn*".c" : fn # workaround #176 - fid = jldopen(fnc, "w", compress=compress) + fid = jldopen(fn, "w", compress=compress) @write fid x @write fid A @write fid Aarray @@ -322,9 +327,7 @@ for compress in (true,false) @write fid sym @write fid syms @write fid d - if compress # work around #176 - @write fid ex - end + @write fid ex @write fid T @write fid char @write fid unicode_char @@ -390,6 +393,7 @@ for compress in (true,false) @write fid bitsparamint @write fid bitsparamuint @write fid tuple_of_tuples + VERSION >= v"0.4.0-dev+4319" && @write fid simplevec # Make sure we can create groups (i.e., use HDF5 features) g = g_create(fid, "mygroup") @@ -401,7 +405,7 @@ for compress in (true,false) # mmapping currently fails on Windows; re-enable once it can work for mmap = (@windows ? false : (false, true)) - fidr = jldopen(fnc, "r", mmaparrays=mmap) + fidr = jldopen(fn, "r", mmaparrays=mmap) @check fidr x @check fidr A dsetA = fidr["A"] @@ -433,10 +437,8 @@ for compress in (true,false) @check fidr sym @check fidr syms @check fidr d - if compress # work around #176 - exr = read(fidr, "ex") # line numbers are stripped, don't expect equality - checkexpr(ex, exr) - end + exr = read(fidr, "ex") # line numbers are stripped, don't expect equality + checkexpr(ex, exr) @check fidr T @check fidr char @check fidr unicode_char @@ -520,6 +522,7 @@ for compress in (true,false) @check fidr bitsparamint @check fidr bitsparamuint @check fidr tuple_of_tuples + VERSION >= v"0.4.0-dev+4319" && @check fidr simplevec x1 = read(fidr, "group1/x") @assert x1 == Any[1] @@ -734,13 +737,13 @@ jldopen(fn, "r") do file for i = 1:5 @test x[i].x.x == i end - @test typeof(read(file, "x6")).names == () + @test isempty(fieldnames(typeof(read(file, "x6")))) @test reinterpret(UInt8, read(file, "x7")) == 0x77 x = read(file, "x8") @test x.a.x == 2 @test x.b.x.x == 3 - @test typeof(x.c).names == () + @test isempty(fieldnames(typeof(x.c))) @test reinterpret(UInt8, x.d) == 0x12 x = read(file, "x9") @@ -753,7 +756,7 @@ jldopen(fn, "r") do file for i = 1:5 @test x[2][2][i].x.x == i end - @test typeof(x[3]).names == () + @test isempty(fieldnames(typeof(x[3]))) end # Issue #176 diff --git a/test/plain.jl b/test/plain.jl index 332913e53..0cee20f04 100644 --- a/test/plain.jl +++ b/test/plain.jl @@ -1,4 +1,4 @@ -using HDF5, Compat +using HDF5, Compat, Base.Test const test_path = splitdir(@__FILE__)[1] # Create a new file @@ -187,7 +187,7 @@ end # Test reading multiple vars at once z = read(fr, "Float64", "Int16") @assert z == (3.2, 4) -@assert typeof(z) == (Float64, Int16) +@assert typeof(z) == @compat Tuple{Float64, Int16} # Test function syntax read(fr, "Float64") do x @assert x == 3.2 @@ -232,6 +232,6 @@ d = h5read(joinpath(test_path, "compound.h5"), "/data") @assert typeof(d) == HDF5.HDF5Compound @assert typeof(d.data) == Array{UInt8,1} @assert length(d.data) == 128 -@assert d.membertype == Type[Float64, HDF5.FixedArray{Float64,(HDF5.DimSize{3},)}, HDF5.FixedArray{Float64,(HDF5.DimSize{3},)}, Float64] +@test d.membertype == Type[Float64, HDF5.FixedArray{Float64,(3,)}, HDF5.FixedArray{Float64,(3,)}, Float64] @assert d.membername == ASCIIString["wgt", "xyz", "uvw", "E"] @assert d.memberoffset == UInt64[0x00, 0x08, 0x20, 0x38]