Skip to content
Permalink
Browse files
implement Base.IteratorSize for Stream, fixes #141 (#142)
  • Loading branch information
damiendr committed Mar 5, 2021
1 parent a113edd commit 8e7869df7029ff544443395136e8b35f38f05158
Show file tree
Hide file tree
Showing 2 changed files with 8 additions and 3 deletions.
@@ -84,6 +84,8 @@ function Stream(bytes::Vector{UInt8}, off::Integer=1, tlen::Union{Integer, Nothi
return Stream(batchiterator, pos, names, schema, dictencodings, dictencoded, convert)
end

Base.IteratorSize(::Type{Stream}) = Base.SizeUnknown()

function Base.iterate(x::Stream, (pos, id)=(x.pos, 1))
columns = AbstractVector[]
while true
@@ -54,13 +54,13 @@ end # @testset "arrow json integration tests"
@testset "misc" begin

# multiple record batches
t = Tables.partitioner(((col1=Union{Int64, Missing}[1,2,3,4,5,6,7,8,9,missing],), (col1=Union{Int64, Missing}[1,2,3,4,5,6,7,8,9,missing],)))
t = Tables.partitioner(((col1=Union{Int64, Missing}[1,2,3,4,5,6,7,8,9,missing],), (col1=Union{Int64, Missing}[missing,11],)))
io = IOBuffer()
Arrow.write(io, t)
seekstart(io)
tt = Arrow.Table(io)
@test length(tt) == 1
@test isequal(tt.col1, vcat([1,2,3,4,5,6,7,8,9,missing], [1,2,3,4,5,6,7,8,9,missing]))
@test isequal(tt.col1, vcat([1,2,3,4,5,6,7,8,9,missing], [missing,11]))
@test eltype(tt.col1) === Union{Int64, Missing}

# Arrow.Stream
@@ -76,10 +76,13 @@ state = iterate(str, st)
@test state !== nothing
tt, st = state
@test length(tt) == 1
@test isequal(tt.col1, [1,2,3,4,5,6,7,8,9,missing])
@test isequal(tt.col1, [missing,11])

@test iterate(str, st) === nothing

@test isequal(collect(str)[1].col1, [1,2,3,4,5,6,7,8,9,missing])
@test isequal(collect(str)[2].col1, [missing,11])

# dictionary batch isDelta
t = (
col1=Int64[1,2,3,4],

0 comments on commit 8e7869d

Please sign in to comment.