Skip to content
Browse files
Warn when converting Arrow.Timestamps to Dates.DateTime or ZonedDateT…
…ime (#172)

Fixes #166. The problem OP saw in the original issue was that we didn't
have a proper `ArrowTypes.fromarrow` method defined for `Dates.DateTime`
from `Arrow.Timestamp` with nanosecond precision, which is accurate in
one sense because `Dates.DateTime` only supports up to millisecond
precision. But better than just erroring when trying to access these
values later, we now do the conversion anyway, which may be lossy, and
issue a warning about the potentially lossy conversion. If > millisecond
precision is needed, then users should pass `convert=false` and operate
on the `Arrow.Timestamp` values directly for now.
  • Loading branch information
quinnj committed Apr 13, 2021
1 parent 4876810 commit 2909821738a2d49b7a540764dae93d6fc88e068b
Showing 2 changed files with 22 additions and 5 deletions.
@@ -260,10 +260,20 @@ const DATETIME = Timestamp{Meta.TimeUnit.MILLISECOND, nothing}

finaljuliatype(::Type{Timestamp{U, TZ}}) where {U, TZ} = ZonedDateTime
finaljuliatype(::Type{Timestamp{U, nothing}}) where {U} = DateTime
Base.convert(::Type{ZonedDateTime}, x::Timestamp{U, TZ}) where {U, TZ} =
ZonedDateTime(Dates.DateTime(Dates.UTM(Int64(Dates.toms(periodtype(U)(x.x)) + UNIX_EPOCH_DATETIME))), TimeZone(String(TZ)))
Base.convert(::Type{DateTime}, x::Timestamp{U, nothing}) where {U} =
Dates.DateTime(Dates.UTM(Int64(Dates.toms(periodtype(U)(x.x)) + UNIX_EPOCH_DATETIME)))

@noinline warntimestamp(U, T) =
@warn "automatically converting Arrow.Timestamp with precision = $U to `$T` which only supports millisecond precision; conversion may be lossy; to avoid converting, pass `Arrow.Table(source; convert=false)" maxlog=1

function Base.convert(::Type{ZonedDateTime}, x::Timestamp{U, TZ}) where {U, TZ}
(U === Meta.TimeUnit.MICROSECOND || U == Meta.TimeUnit.NANOSECOND) && warntimestamp(U, ZonedDateTime)
return ZonedDateTime(Dates.DateTime(Dates.UTM(Int64(Dates.toms(periodtype(U)(x.x)) + UNIX_EPOCH_DATETIME))), TimeZone(String(TZ)))

function Base.convert(::Type{DateTime}, x::Timestamp{U, nothing}) where {U}
(U === Meta.TimeUnit.MICROSECOND || U == Meta.TimeUnit.NANOSECOND) && warntimestamp(U, DateTime)
return Dates.DateTime(Dates.UTM(Int64(Dates.toms(periodtype(U)(x.x)) + UNIX_EPOCH_DATETIME)))

Base.convert(::Type{Timestamp{Meta.TimeUnit.MILLISECOND, TZ}}, x::ZonedDateTime) where {TZ} =
Timestamp{Meta.TimeUnit.MILLISECOND, TZ}(Int64(Dates.value(DateTime(x)) - UNIX_EPOCH_DATETIME))
Base.convert(::Type{Timestamp{Meta.TimeUnit.MILLISECOND, nothing}}, x::DateTime) =
@@ -282,7 +292,7 @@ ArrowTypes.toarrow(x::Dates.DateTime) = convert(DATETIME, x)
const DATETIME_SYMBOL = Symbol("JuliaLang.DateTime")
ArrowTypes.arrowname(::Type{Dates.DateTime}) = DATETIME_SYMBOL
ArrowTypes.JuliaType(::Val{DATETIME_SYMBOL}, S) = Dates.DateTime
ArrowTypes.fromarrow(::Type{Dates.DateTime}, x::DATETIME) = convert(Dates.DateTime, x)
ArrowTypes.fromarrow(::Type{Dates.DateTime}, x::Timestamp) = convert(Dates.DateTime, x)
ArrowTypes.default(::Type{Dates.DateTime}) = Dates.DateTime(1,1,1,1,1,1)

ArrowTypes.ArrowType(::Type{ZonedDateTime}) = Timestamp
@@ -282,6 +282,13 @@ m = Dict("a" => "b")
Arrow.setmetadata!(tbl, m)
@test Arrow.getmetadata(tbl) === m

# 166
t = (
col1=[zero(Arrow.Timestamp{Arrow.Meta.TimeUnit.NANOSECOND, nothing})],
tbl = Arrow.Table(Arrow.tobuffer(t))
@test tbl.col1[1] == Dates.DateTime(1970)

end # @testset "misc"


0 comments on commit 2909821

Please sign in to comment.