Skip to content

Commit

Permalink
Separate low-level and high-level mmap wrappers to separate functions
Browse files Browse the repository at this point in the history 
This cleans up the method list for the intended user-facing `mmap`
function just a little bit --- namely, the function signature
involving a `RawFD` argument is no longer shown.

Simultaneously, we slightly relax the type constraints on all of the
high-level `mmap` methods to accept any `<:Array{T}` type so that
both specific-shape arrays (i.e. `Vector{T}`, `Matrix{T}`) and the
generic `Array{T}` argument can be used in user code to do the same
thing.

(Only the number of dimensions arguments matters in the end, so the
confusing `UnixMmap.mmap(Vector{Float64}, (3, 64))` actually returns a
matrix, but I find that less objectionable than not permitting
`UnixMmap.mmap(Matrix{Float64}, (3, 64))` to even work.)
  • Loading branch information
@jmert
jmert committed Dec 21, 2021
1 parent 4cb7be9 commit 1c46181
Show file tree
Hide file tree
Showing 4 changed files with 65 additions and 23 deletions.
6 changes: 3 additions & 3 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,15 +19,15 @@ julia> using UnixMmap
A file can be memory mapped (read-only by default) by giving the filename and the `Array`
type (optionally with dimensions to give a shape):
```julia
julia> UnixMmap.mmap("arbitrary.dat", Array{Float64})
julia> UnixMmap.mmap("arbitrary.dat", Vector{Float64})
192-element Vector{Float64}:
0.0
0.0
⋮
0.0
0.0

julia> UnixMmap.mmap("arbitrary.dat", Array{Float64}, (64, 3))
julia> UnixMmap.mmap("arbitrary.dat", Matrix{Float64}, (64, 3))
64×3 Matrix{Float64}:
0.0 0.0 0.0
0.0 0.0 0.0
Expand All @@ -38,7 +38,7 @@ julia> UnixMmap.mmap("arbitrary.dat", Array{Float64}, (64, 3))
while an anonymous memory map can be created by instead specifying the `Array` type and
dimensions:
```julia
julia> UnixMmap.mmap(Array{Float64}, (128, 3))
julia> UnixMmap.mmap(Matrix{Float64}, (128, 3))
128×3 Matrix{Float64}:
0.0 0.0 0.0
0.0 0.0 0.0
Expand Down
8 changes: 4 additions & 4 deletions docs/src/index.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,15 +11,15 @@ A file can be memory mapped (read-only by default) by calling [`UnixMmap.mmap`](
with a filename and the `Array` type to be applied (and optionally with dimensions to give a
shape):
```julia-repl
julia> UnixMmap.mmap("arbitrary.dat", Array{Float64})
julia> UnixMmap.mmap("arbitrary.dat", Vector{Float64})
192-element Vector{Float64}:
0.0
0.0
⋮
0.0
0.0
julia> UnixMmap.mmap("arbitrary.dat", Array{Float64}, (64, 3))
julia> UnixMmap.mmap("arbitrary.dat", Matrix{Float64}, (64, 3))
64×3 Matrix{Float64}:
0.0 0.0 0.0
0.0 0.0 0.0
Expand All @@ -30,7 +30,7 @@ julia> UnixMmap.mmap("arbitrary.dat", Array{Float64}, (64, 3))
while an anonymous memory map can be created by instead specifying the `Array` type and
dimensions:
```julia-repl
julia> UnixMmap.mmap(Array{Float64}, (128, 3))
julia> UnixMmap.mmap(Matrix{Float64}, (128, 3))
128×3 Matrix{Float64}:
0.0 0.0 0.0
0.0 0.0 0.0
Expand All @@ -44,7 +44,7 @@ is the ability to set Unix-specific flags during mapping.
For example, on Linux the `MAP_POPULATE` flag can be used to advise the kernel to
prefault all mapped pages into active memory.
```julia-repl
julia> UnixMmap.mmap("arbitrary.dat", Array{Float64}, (64, 3);
julia> UnixMmap.mmap("arbitrary.dat", Matrix{Float64}, (64, 3);
flags = UnixMmap.MAP_SHARED | UnixMmap.MAP_POPULATE)
64×3 Matrix{Float64}:
0.0 0.0 0.0
Expand Down
32 changes: 16 additions & 16 deletions src/mmap.jl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,7 @@ fileflags(::IO) = MAP_SHARED
###

# Raw call to mmap syscall
function _mmap(ptr::Ptr{Cvoid}, len::Int,
function _sys_mmap(ptr::Ptr{Cvoid}, len::Int,
prot::MmapProtection, flags::MmapFlags,
fd::RawFD, offset::Int64)

Expand All @@ -53,7 +53,7 @@ function _mmap(ptr::Ptr{Cvoid}, len::Int,
return ret
end

function _unmap!(ptr::Ptr{Cvoid}, len::Int)
function _sys_unmap!(ptr::Ptr{Cvoid}, len::Int)
ret = ccall(:munmap, Cint, (Ptr{Cvoid}, Csize_t), ptr, len)
Base.systemerror("munmap", ret != 0)
return
Expand All @@ -62,7 +62,7 @@ end
# Low-level form which mirrors a raw mmap, but constructs a Julia array of given
# dimension(s) at a specific offset within a file (includes accounting for page alignment
# requirement).
function mmap(::Type{Array{T}}, dims::NTuple{N,Integer},
function _mmap(::Type{Array{T}}, dims::NTuple{N,Integer},
prot::MmapProtection, flags::MmapFlags,
fd::RawFD, offset::Integer) where {T, N}
isbitstype(T) || throw(ArgumentError("unable to mmap type $T; must satisfy `isbitstype(T) == true`"))
Expand All @@ -75,19 +75,19 @@ function mmap(::Type{Array{T}}, dims::NTuple{N,Integer},
page_pad = rem(Int64(offset), PAGESIZE)
mmaplen::Int = len + page_pad

ptr = _mmap(C_NULL, mmaplen, prot, flags, fd, Int64(offset) - page_pad)
ptr = _sys_mmap(C_NULL, mmaplen, prot, flags, fd, Int64(offset) - page_pad)
aptr = convert(Ptr{T}, ptr + page_pad)
array = unsafe_wrap(Array{T,N}, aptr, dims)
finalizer(_ -> _unmap!(ptr, mmaplen), array)
finalizer(_ -> _sys_unmap!(ptr, mmaplen), array)
return array
end
function mmap(::Type{Array{T}}, len::Int, prot::MmapProtection, flags::MmapFlags,
function _mmap(::Type{Array{T}}, len::Int, prot::MmapProtection, flags::MmapFlags,
fd::RawFD, offset::Int) where {T}
return mmap(Array{T}, (Int(len),), prot, flags, fd, offset)
return _mmap(Array{T}, (Int(len),), prot, flags, fd, offset)
end

# Higher-level interface which takes an IO object and sets default flag values.
function mmap(io::IO, ::Type{Array{T}}, dims::NTuple{N,Integer};
function mmap(io::IO, ::Type{<:Array{T}}, dims::NTuple{N,Integer};
offset::Union{Integer,Nothing} = nothing,
prot::Union{MmapProtection,Nothing} = nothing,
flags::Union{MmapFlags,Nothing} = nothing,
Expand All @@ -107,9 +107,9 @@ function mmap(io::IO, ::Type{Array{T}}, dims::NTuple{N,Integer};

grow && iswritable(io) && grow!(io, offset, len)

return mmap(Array{T}, dims, prot, flags, gethandle(io), offset)
return _mmap(Array{T}, dims, prot, flags, gethandle(io), offset)
end
function mmap(io::IO, ::Type{Array{T}}, len::Integer;
function mmap(io::IO, ::Type{<:Array{T}}, len::Integer;
offset::Union{Integer,Nothing} = nothing,
prot::Union{MmapProtection,Nothing} = nothing,
flags::Union{MmapFlags,Nothing} = nothing,
Expand All @@ -118,7 +118,7 @@ function mmap(io::IO, ::Type{Array{T}}, len::Integer;
return mmap(io, Array{T}, (len,);
offset = offset, prot = prot, flags = flags, grow = grow)
end
function mmap(io::IO, ::Type{Array{T}} = Array{UInt8};
function mmap(io::IO, ::Type{<:Array{T}} = Array{UInt8};
offset::Union{Integer,Nothing} = nothing,
prot::Union{MmapProtection,Nothing} = nothing,
flags::Union{MmapFlags,Nothing} = nothing,
Expand All @@ -129,7 +129,7 @@ function mmap(io::IO, ::Type{Array{T}} = Array{UInt8};
end

# Mapping of files
function mmap(file::AbstractString, ::Type{Array{T}}, dims::NTuple{N,Integer};
function mmap(file::AbstractString, ::Type{<:Array{T}}, dims::NTuple{N,Integer};
offset::Union{Integer,Nothing} = nothing,
prot::Union{MmapProtection,Nothing} = nothing,
flags::Union{MmapFlags,Nothing} = nothing,
Expand All @@ -154,7 +154,7 @@ function mmap(file::AbstractString, ::Type{Array{T}}, dims::NTuple{N,Integer};
offset = offset, prot = prot, flags = flags, grow = grow)
end
end
function mmap(file::AbstractString, ::Type{Array{T}}, len::Integer;
function mmap(file::AbstractString, ::Type{<:Array{T}}, len::Integer;
offset::Union{Integer,Nothing} = nothing,
prot::Union{MmapProtection,Nothing} = nothing,
flags::Union{MmapFlags,Nothing} = nothing,
Expand All @@ -164,7 +164,7 @@ function mmap(file::AbstractString, ::Type{Array{T}}, len::Integer;
offset = offset, prot = prot, flags = flags, grow = grow)
end
# Default mapping of the [rest of] given file
function mmap(file::AbstractString, ::Type{Array{T}} = Array{UInt8};
function mmap(file::AbstractString, ::Type{<:Array{T}} = Array{UInt8};
offset::Union{Integer,Nothing} = nothing,
prot::Union{MmapProtection,Nothing} = nothing,
flags::Union{MmapFlags,Nothing} = nothing,
Expand All @@ -175,7 +175,7 @@ function mmap(file::AbstractString, ::Type{Array{T}} = Array{UInt8};
end

# form to construct anonymous memory maps
function mmap(::Type{Array{T}}, dims::NTuple{N,Integer};
function mmap(::Type{<:Array{T}}, dims::NTuple{N,Integer};
prot::Union{MmapProtection,Nothing} = nothing,
flags::Union{MmapFlags,Nothing} = nothing
) where {T, N}
Expand All @@ -184,7 +184,7 @@ function mmap(::Type{Array{T}}, dims::NTuple{N,Integer};
return mmap(Anonymous(), Array{T}, dims;
offset = Int64(0), prot = prot, flags = flags, grow = false)
end
function mmap(::Type{Array{T}}, len::Integer;
function mmap(::Type{<:Array{T}}, len::Integer;
prot::Union{MmapProtection,Nothing} = nothing,
flags::Union{MmapFlags,Nothing} = nothing
) where {T}
Expand Down
42 changes: 42 additions & 0 deletions test/runtests.jl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -169,6 +169,48 @@ end
end
end

@testset "Inferring return types" begin
# N.B. also tests that any <:Array{T} works as the array type argument
data = repeat(1.0:16.0, 1, 3)
mktemp() do path, io
write(io, data)
flush(io)
seek(io, 0)
@testset "IO handle - $AT" for AT in (Array, Vector, Matrix)
# dims isa Tuple{Int}, so V isa Vector no matter AT
V = @inferred mmap(io, AT{Float64}, (16*3,))
@test V isa Vector
@test V == vec(data)
# dims isa Tuple{Int,Int}, so M isa Matrix no matter AT
M = @inferred mmap(io, AT{Float64}, (16, 3))
@test M isa Matrix
@test M == data
end
close(io)
@testset "File name - $AT" for AT in (Array, Vector, Matrix)
# dims isa Tuple{Int}, so V isa Vector no matter AT
V = @inferred mmap(path, AT{Float64}, (16*3,))
@test V isa Vector
@test V == vec(data)
# dims isa Tuple{Int,Int}, so M isa Matrix no matter AT
M = @inferred mmap(path, AT{Float64}, (16, 3))
@test M isa Matrix
@test M == data
end
@testset "Anonymous - $AT" for AT in (Array, Vector, Matrix)
# dims isa Tuple{Int}, so V isa Vector no matter AT
V = @inferred mmap(AT{Float64}, (16*3,))
@test V isa Vector
@test all(iszero, V)
# dims isa Tuple{Int,Int}, so M isa Matrix no matter AT
M = @inferred mmap(AT{Float64}, (16, 3))
@test M isa Matrix
@test all(iszero, M)
end
GC.gc()
end
end

@testset "Anonymous memory maps" begin
A = mmap(Array{Int16}, 500)
@test size(A) == (500,)
Expand Down

0 comments on commit 1c46181

Please sign in to comment.