From cabceee561c9d6252014f0e31b74c08b44cb88fd Mon Sep 17 00:00:00 2001 From: Justin Willmert Date: Sat, 18 Dec 2021 11:51:27 -0600 Subject: [PATCH 1/2] Add example usages in documentation --- Project.toml | 2 +- README.md | 46 ++++++++++++++++++++++++----------- docs/src/index.md | 61 +++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 94 insertions(+), 15 deletions(-) diff --git a/Project.toml b/Project.toml index 717f9ef..2b76183 100644 --- a/Project.toml +++ b/Project.toml @@ -1,7 +1,7 @@ name = "UnixMmap" uuid = "674b2976-56af-439b-a2b1-837be4a3d087" authors = ["Justin Willmert and contributors"] -version = "0.1.1" +version = "1.0.0" [deps] BitFlags = "d1d4a3ce-64b1-5f1a-9ba4-7e7e69966f35" diff --git a/README.md b/README.md index 7705827..9abe71b 100644 --- a/README.md +++ b/README.md @@ -9,24 +9,42 @@ exposing the Unix memory-mapping interface. ### Installation and usage -This library is **not** registered in Julia's [General registry][General.jl], -so the package must be installed either by cloning it directly: +Installation and loading is as easy as: +```julia +pkg> add UnixMmap -``` -(@v1.6) pkg> add https://github.com/jmert/UnixMmap.jl -``` - -or by making use of my [personal registry][Registry.jl]: - -``` -(@v1.6) pkg> registry add https://github.com/jmert/Registry.jl -(@v1.6) pkg> add UnixMmap +julia> using UnixMmap ``` -After installing, just load like any other Julia package: - +A file can be memory mapped (read-only by default) by giving the filename and the `Array` +type (optionally with dimensions to give a shape): +```julia +julia> UnixMmap.mmap("arbitrary.dat", Array{Float64}) +192-element Vector{Float64}: + 0.0 + 0.0 + ⋮ + 0.0 + 0.0 + +julia> UnixMmap.mmap("arbitrary.dat", Array{Float64}, (64, 3)) +64×3 Matrix{Float64}: + 0.0 0.0 0.0 + 0.0 0.0 0.0 + ⋮ + 0.0 0.0 0.0 + 0.0 0.0 0.0 ``` -julia> using UnixMmap +while an anonymous memory map can be created by instead specifying the `Array` type and +dimensions: +```julia +julia> UnixMmap.mmap(Array{Float64}, (128, 3)) +128×3 Matrix{Float64}: + 0.0 0.0 0.0 + 0.0 0.0 0.0 + ⋮ + 0.0 0.0 0.0 + 0.0 0.0 0.0 ``` [docs-stable-img]: https://img.shields.io/badge/docs-stable-blue.svg diff --git a/docs/src/index.md b/docs/src/index.md index 7282bf5..42b993b 100644 --- a/docs/src/index.md +++ b/docs/src/index.md @@ -1,5 +1,66 @@ # UnixMmap.jl Documentation +Installation and loading is as easy as: +```julia-repl +pkg> add UnixMmap + +julia> using UnixMmap +``` + +A file can be memory mapped (read-only by default) by calling [`UnixMmap.mmap`](@ref) +with a filename and the `Array` type to be applied (and optionally with dimensions to give a +shape): +```julia-repl +julia> UnixMmap.mmap("arbitrary.dat", Array{Float64}) +192-element Vector{Float64}: + 0.0 + 0.0 + ⋮ + 0.0 + 0.0 + +julia> UnixMmap.mmap("arbitrary.dat", Array{Float64}, (64, 3)) +64×3 Matrix{Float64}: + 0.0 0.0 0.0 + 0.0 0.0 0.0 + ⋮ + 0.0 0.0 0.0 + 0.0 0.0 0.0 +``` +while an anonymous memory map can be created by instead specifying the `Array` type and +dimensions: +```julia-repl +julia> UnixMmap.mmap(Array{Float64}, (128, 3)) +128×3 Matrix{Float64}: + 0.0 0.0 0.0 + 0.0 0.0 0.0 + ⋮ + 0.0 0.0 0.0 + 0.0 0.0 0.0 +``` + +The notable features that UnixMmap.jl provides over the standard library's Mmap module +is the ability to set Unix-specific flags during mapping. +For example, on Linux the `MAP_POPULATE` flag can be used to advise the kernel to +prefault all mapped pages into active memory. +```julia-repl +julia> UnixMmap.mmap("arbitrary.dat", Array{Float64}, (64, 3); + flags = UnixMmap.MAP_SHARED | UnixMmap.MAP_POPULATE) +64×3 Matrix{Float64}: + 0.0 0.0 0.0 + 0.0 0.0 0.0 + ⋮ + 0.0 0.0 0.0 + 0.0 0.0 0.0 +``` + +UnixMmap.jl provides OS-specific flags for several Unixes; see the +[Constants](@ref Constants-—-Linux) section for more details. + +The package also exposes the [`UnixMmap.madvise!`](@ref), [`UnixMmap.msync!`](@ref), and +[`UnixMmap.mincore`](@ref) functions which correspond closely to the underlying system +calls. + ## Library API Reference ```@contents Pages = [ From 22b93181907364891f4c7487b9a854e2cd8617e8 Mon Sep 17 00:00:00 2001 From: Justin Willmert Date: Sat, 18 Dec 2021 12:04:29 -0600 Subject: [PATCH 2/2] Separate low-level and high-level mmap wrappers to separate functions This cleans up the method list for the intended user-facing `mmap` function just a little bit --- namely, the function signature involving a `RawFD` argument is no longer shown. Simultaneously, we slightly relax the type constraints on all of the high-level `mmap` methods to accept any `<:Array{T}` type so that both specific-shape arrays (i.e. `Vector{T}`, `Matrix{T}`) and the generic `Array{T}` argument can be used in user code to do the same thing. (Only the number of dimensions arguments matters in the end, so the confusing `UnixMmap.mmap(Vector{Float64}, (3, 64))` actually returns a matrix, but I find that less objectionable than not permitting `UnixMmap.mmap(Matrix{Float64}, (3, 64))` to even work.) --- README.md | 6 +++--- docs/src/index.md | 8 ++++---- src/mmap.jl | 32 ++++++++++++++++---------------- test/runtests.jl | 42 ++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 65 insertions(+), 23 deletions(-) diff --git a/README.md b/README.md index 9abe71b..5268388 100644 --- a/README.md +++ b/README.md @@ -19,7 +19,7 @@ julia> using UnixMmap A file can be memory mapped (read-only by default) by giving the filename and the `Array` type (optionally with dimensions to give a shape): ```julia -julia> UnixMmap.mmap("arbitrary.dat", Array{Float64}) +julia> UnixMmap.mmap("arbitrary.dat", Vector{Float64}) 192-element Vector{Float64}: 0.0 0.0 @@ -27,7 +27,7 @@ julia> UnixMmap.mmap("arbitrary.dat", Array{Float64}) 0.0 0.0 -julia> UnixMmap.mmap("arbitrary.dat", Array{Float64}, (64, 3)) +julia> UnixMmap.mmap("arbitrary.dat", Matrix{Float64}, (64, 3)) 64×3 Matrix{Float64}: 0.0 0.0 0.0 0.0 0.0 0.0 @@ -38,7 +38,7 @@ julia> UnixMmap.mmap("arbitrary.dat", Array{Float64}, (64, 3)) while an anonymous memory map can be created by instead specifying the `Array` type and dimensions: ```julia -julia> UnixMmap.mmap(Array{Float64}, (128, 3)) +julia> UnixMmap.mmap(Matrix{Float64}, (128, 3)) 128×3 Matrix{Float64}: 0.0 0.0 0.0 0.0 0.0 0.0 diff --git a/docs/src/index.md b/docs/src/index.md index 42b993b..3832cd5 100644 --- a/docs/src/index.md +++ b/docs/src/index.md @@ -11,7 +11,7 @@ A file can be memory mapped (read-only by default) by calling [`UnixMmap.mmap`]( with a filename and the `Array` type to be applied (and optionally with dimensions to give a shape): ```julia-repl -julia> UnixMmap.mmap("arbitrary.dat", Array{Float64}) +julia> UnixMmap.mmap("arbitrary.dat", Vector{Float64}) 192-element Vector{Float64}: 0.0 0.0 @@ -19,7 +19,7 @@ julia> UnixMmap.mmap("arbitrary.dat", Array{Float64}) 0.0 0.0 -julia> UnixMmap.mmap("arbitrary.dat", Array{Float64}, (64, 3)) +julia> UnixMmap.mmap("arbitrary.dat", Matrix{Float64}, (64, 3)) 64×3 Matrix{Float64}: 0.0 0.0 0.0 0.0 0.0 0.0 @@ -30,7 +30,7 @@ julia> UnixMmap.mmap("arbitrary.dat", Array{Float64}, (64, 3)) while an anonymous memory map can be created by instead specifying the `Array` type and dimensions: ```julia-repl -julia> UnixMmap.mmap(Array{Float64}, (128, 3)) +julia> UnixMmap.mmap(Matrix{Float64}, (128, 3)) 128×3 Matrix{Float64}: 0.0 0.0 0.0 0.0 0.0 0.0 @@ -44,7 +44,7 @@ is the ability to set Unix-specific flags during mapping. For example, on Linux the `MAP_POPULATE` flag can be used to advise the kernel to prefault all mapped pages into active memory. ```julia-repl -julia> UnixMmap.mmap("arbitrary.dat", Array{Float64}, (64, 3); +julia> UnixMmap.mmap("arbitrary.dat", Matrix{Float64}, (64, 3); flags = UnixMmap.MAP_SHARED | UnixMmap.MAP_POPULATE) 64×3 Matrix{Float64}: 0.0 0.0 0.0 diff --git a/src/mmap.jl b/src/mmap.jl index 88dee93..27c6a16 100644 --- a/src/mmap.jl +++ b/src/mmap.jl @@ -39,7 +39,7 @@ fileflags(::IO) = MAP_SHARED ### # Raw call to mmap syscall -function _mmap(ptr::Ptr{Cvoid}, len::Int, +function _sys_mmap(ptr::Ptr{Cvoid}, len::Int, prot::MmapProtection, flags::MmapFlags, fd::RawFD, offset::Int64) @@ -53,7 +53,7 @@ function _mmap(ptr::Ptr{Cvoid}, len::Int, return ret end -function _unmap!(ptr::Ptr{Cvoid}, len::Int) +function _sys_unmap!(ptr::Ptr{Cvoid}, len::Int) ret = ccall(:munmap, Cint, (Ptr{Cvoid}, Csize_t), ptr, len) Base.systemerror("munmap", ret != 0) return @@ -62,7 +62,7 @@ end # Low-level form which mirrors a raw mmap, but constructs a Julia array of given # dimension(s) at a specific offset within a file (includes accounting for page alignment # requirement). -function mmap(::Type{Array{T}}, dims::NTuple{N,Integer}, +function _mmap(::Type{Array{T}}, dims::NTuple{N,Integer}, prot::MmapProtection, flags::MmapFlags, fd::RawFD, offset::Integer) where {T, N} isbitstype(T) || throw(ArgumentError("unable to mmap type $T; must satisfy `isbitstype(T) == true`")) @@ -75,19 +75,19 @@ function mmap(::Type{Array{T}}, dims::NTuple{N,Integer}, page_pad = rem(Int64(offset), PAGESIZE) mmaplen::Int = len + page_pad - ptr = _mmap(C_NULL, mmaplen, prot, flags, fd, Int64(offset) - page_pad) + ptr = _sys_mmap(C_NULL, mmaplen, prot, flags, fd, Int64(offset) - page_pad) aptr = convert(Ptr{T}, ptr + page_pad) array = unsafe_wrap(Array{T,N}, aptr, dims) - finalizer(_ -> _unmap!(ptr, mmaplen), array) + finalizer(_ -> _sys_unmap!(ptr, mmaplen), array) return array end -function mmap(::Type{Array{T}}, len::Int, prot::MmapProtection, flags::MmapFlags, +function _mmap(::Type{Array{T}}, len::Int, prot::MmapProtection, flags::MmapFlags, fd::RawFD, offset::Int) where {T} - return mmap(Array{T}, (Int(len),), prot, flags, fd, offset) + return _mmap(Array{T}, (Int(len),), prot, flags, fd, offset) end # Higher-level interface which takes an IO object and sets default flag values. -function mmap(io::IO, ::Type{Array{T}}, dims::NTuple{N,Integer}; +function mmap(io::IO, ::Type{<:Array{T}}, dims::NTuple{N,Integer}; offset::Union{Integer,Nothing} = nothing, prot::Union{MmapProtection,Nothing} = nothing, flags::Union{MmapFlags,Nothing} = nothing, @@ -107,9 +107,9 @@ function mmap(io::IO, ::Type{Array{T}}, dims::NTuple{N,Integer}; grow && iswritable(io) && grow!(io, offset, len) - return mmap(Array{T}, dims, prot, flags, gethandle(io), offset) + return _mmap(Array{T}, dims, prot, flags, gethandle(io), offset) end -function mmap(io::IO, ::Type{Array{T}}, len::Integer; +function mmap(io::IO, ::Type{<:Array{T}}, len::Integer; offset::Union{Integer,Nothing} = nothing, prot::Union{MmapProtection,Nothing} = nothing, flags::Union{MmapFlags,Nothing} = nothing, @@ -118,7 +118,7 @@ function mmap(io::IO, ::Type{Array{T}}, len::Integer; return mmap(io, Array{T}, (len,); offset = offset, prot = prot, flags = flags, grow = grow) end -function mmap(io::IO, ::Type{Array{T}} = Array{UInt8}; +function mmap(io::IO, ::Type{<:Array{T}} = Array{UInt8}; offset::Union{Integer,Nothing} = nothing, prot::Union{MmapProtection,Nothing} = nothing, flags::Union{MmapFlags,Nothing} = nothing, @@ -129,7 +129,7 @@ function mmap(io::IO, ::Type{Array{T}} = Array{UInt8}; end # Mapping of files -function mmap(file::AbstractString, ::Type{Array{T}}, dims::NTuple{N,Integer}; +function mmap(file::AbstractString, ::Type{<:Array{T}}, dims::NTuple{N,Integer}; offset::Union{Integer,Nothing} = nothing, prot::Union{MmapProtection,Nothing} = nothing, flags::Union{MmapFlags,Nothing} = nothing, @@ -154,7 +154,7 @@ function mmap(file::AbstractString, ::Type{Array{T}}, dims::NTuple{N,Integer}; offset = offset, prot = prot, flags = flags, grow = grow) end end -function mmap(file::AbstractString, ::Type{Array{T}}, len::Integer; +function mmap(file::AbstractString, ::Type{<:Array{T}}, len::Integer; offset::Union{Integer,Nothing} = nothing, prot::Union{MmapProtection,Nothing} = nothing, flags::Union{MmapFlags,Nothing} = nothing, @@ -164,7 +164,7 @@ function mmap(file::AbstractString, ::Type{Array{T}}, len::Integer; offset = offset, prot = prot, flags = flags, grow = grow) end # Default mapping of the [rest of] given file -function mmap(file::AbstractString, ::Type{Array{T}} = Array{UInt8}; +function mmap(file::AbstractString, ::Type{<:Array{T}} = Array{UInt8}; offset::Union{Integer,Nothing} = nothing, prot::Union{MmapProtection,Nothing} = nothing, flags::Union{MmapFlags,Nothing} = nothing, @@ -175,7 +175,7 @@ function mmap(file::AbstractString, ::Type{Array{T}} = Array{UInt8}; end # form to construct anonymous memory maps -function mmap(::Type{Array{T}}, dims::NTuple{N,Integer}; +function mmap(::Type{<:Array{T}}, dims::NTuple{N,Integer}; prot::Union{MmapProtection,Nothing} = nothing, flags::Union{MmapFlags,Nothing} = nothing ) where {T, N} @@ -184,7 +184,7 @@ function mmap(::Type{Array{T}}, dims::NTuple{N,Integer}; return mmap(Anonymous(), Array{T}, dims; offset = Int64(0), prot = prot, flags = flags, grow = false) end -function mmap(::Type{Array{T}}, len::Integer; +function mmap(::Type{<:Array{T}}, len::Integer; prot::Union{MmapProtection,Nothing} = nothing, flags::Union{MmapFlags,Nothing} = nothing ) where {T} diff --git a/test/runtests.jl b/test/runtests.jl index bcd345e..92a64f8 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -169,6 +169,48 @@ end end end +@testset "Inferring return types" begin + # N.B. also tests that any <:Array{T} works as the array type argument + data = repeat(1.0:16.0, 1, 3) + mktemp() do path, io + write(io, data) + flush(io) + seek(io, 0) + @testset "IO handle - $AT" for AT in (Array, Vector, Matrix) + # dims isa Tuple{Int}, so V isa Vector no matter AT + V = @inferred mmap(io, AT{Float64}, (16*3,)) + @test V isa Vector + @test V == vec(data) + # dims isa Tuple{Int,Int}, so M isa Matrix no matter AT + M = @inferred mmap(io, AT{Float64}, (16, 3)) + @test M isa Matrix + @test M == data + end + close(io) + @testset "File name - $AT" for AT in (Array, Vector, Matrix) + # dims isa Tuple{Int}, so V isa Vector no matter AT + V = @inferred mmap(path, AT{Float64}, (16*3,)) + @test V isa Vector + @test V == vec(data) + # dims isa Tuple{Int,Int}, so M isa Matrix no matter AT + M = @inferred mmap(path, AT{Float64}, (16, 3)) + @test M isa Matrix + @test M == data + end + @testset "Anonymous - $AT" for AT in (Array, Vector, Matrix) + # dims isa Tuple{Int}, so V isa Vector no matter AT + V = @inferred mmap(AT{Float64}, (16*3,)) + @test V isa Vector + @test all(iszero, V) + # dims isa Tuple{Int,Int}, so M isa Matrix no matter AT + M = @inferred mmap(AT{Float64}, (16, 3)) + @test M isa Matrix + @test all(iszero, M) + end + GC.gc() + end +end + @testset "Anonymous memory maps" begin A = mmap(Array{Int16}, 500) @test size(A) == (500,)