Skip to content
Permalink
Browse files
Add ntasks keyword to limit # of tasks allowed to write at a time (#106)
* Add ntasks keyword to limit # of tasks allowed to write at a time

* fix
  • Loading branch information
quinnj committed Mar 18, 2021
1 parent 8ef9fa4 commit a8aaf95d2dc6baf9a7e015a6baf98ba6221dcf07
Showing 1 changed file with 6 additions and 6 deletions.
@@ -75,18 +75,18 @@ function write end

write(io_or_file; kw...) = x -> write(io_or_file, x; kw...)

function write(file::String, tbl; largelists::Bool=false, compress::Union{Nothing, Symbol, LZ4FrameCompressor, ZstdCompressor}=nothing, denseunions::Bool=true, dictencode::Bool=false, dictencodenested::Bool=false, alignment::Int=8, maxdepth::Int=DEFAULT_MAX_DEPTH)
function write(file::String, tbl; largelists::Bool=false, compress::Union{Nothing, Symbol, LZ4FrameCompressor, ZstdCompressor}=nothing, denseunions::Bool=true, dictencode::Bool=false, dictencodenested::Bool=false, alignment::Int=8, maxdepth::Int=DEFAULT_MAX_DEPTH, ntasks=Inf)
open(file, "w") do io
write(io, tbl, true, largelists, compress, denseunions, dictencode, dictencodenested, alignment, maxdepth)
write(io, tbl, true, largelists, compress, denseunions, dictencode, dictencodenested, alignment, maxdepth, ntasks)
end
return file
end

function write(io::IO, tbl; largelists::Bool=false, compress::Union{Nothing, Symbol, LZ4FrameCompressor, ZstdCompressor}=nothing, denseunions::Bool=true, dictencode::Bool=false, dictencodenested::Bool=false, alignment::Int=8, maxdepth::Int=DEFAULT_MAX_DEPTH, file::Bool=false)
return write(io, tbl, file, largelists, compress, denseunions, dictencode, dictencodenested, alignment, maxdepth)
function write(io::IO, tbl; largelists::Bool=false, compress::Union{Nothing, Symbol, LZ4FrameCompressor, ZstdCompressor}=nothing, denseunions::Bool=true, dictencode::Bool=false, dictencodenested::Bool=false, alignment::Int=8, maxdepth::Int=DEFAULT_MAX_DEPTH, ntasks=Inf, file::Bool=false)
return write(io, tbl, file, largelists, compress, denseunions, dictencode, dictencodenested, alignment, maxdepth, ntasks)
end

function write(io, source, writetofile, largelists, compress, denseunions, dictencode, dictencodenested, alignment, maxdepth)
function write(io, source, writetofile, largelists, compress, denseunions, dictencode, dictencodenested, alignment, maxdepth, ntasks)
if compress === :lz4
compress = LZ4_FRAME_COMPRESSOR
elseif compress === :zstd
@@ -98,7 +98,7 @@ function write(io, source, writetofile, largelists, compress, denseunions, dicte
@debug 1 "starting write of arrow formatted file"
Base.write(io, "ARROW1\0\0")
end
msgs = OrderedChannel{Message}(Inf)
msgs = OrderedChannel{Message}(ntasks)
# build messages
sch = Ref{Tables.Schema}()
firstcols = Ref{Any}()

0 comments on commit a8aaf95

Please sign in to comment.