Merge a8b55bd into cfbca74

Project.toml

@@ -9,17 +9,20 @@ JSON = "682c06a0-de6a-54ab-a142-c8b1cf79cde6"
 Missings = "e1d29d7a-bbdc-5cf2-9ac0-f12de2c33e28"
 Printf = "de0858da-6303-5e67-8744-51eddeeeb8d7"
 Statistics = "10745b16-79ce-11e8-11f9-7d13ad32a3b2"
+StructTypes = "856f2bd8-1eba-4b0a-8007-ebc267875bd4"
 Unicode = "4ec0a83e-493e-50e2-b9ac-8f72acf5a8f5"
 
 [compat]
 DataAPI = "1.1"
 JSON = "0.15, 0.16, 0.17, 0.18, 0.19, 0.20, 0.21"
 Missings = "0.4.3"
+StructTypes = "1"
 julia = "1"
 
 [extras]
 Dates = "ade2ca70-3891-5945-98fb-dc099432e06a"
+JSON3 = "0f8b85d8-7281-11e9-16c2-39a750bddbf1"
 Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
 
 [targets]
-test = ["Dates", "Test"]
+test = ["Dates", "Test", "JSON3"]

src/CategoricalArrays.jl

@@ -12,6 +12,7 @@ module CategoricalArrays
     using DataAPI
     using Missings
     using Printf
+    import StructTypes
 
     include("typedefs.jl")
 

src/array.jl

@@ -193,20 +193,11 @@ CategoricalVector(::UndefInitializer, m::Integer;
                   levels::Union{AbstractVector, Nothing}=nothing,
                   ordered::Bool=false) =
     CategoricalArray(undef, m, levels=levels, ordered=ordered)
-CategoricalVector{T}(::UndefInitializer, m::Int;
-                     levels::Union{AbstractVector, Nothing}=nothing,
-                     ordered::Bool=false) where {T} =
-    CategoricalArray{T}(undef, (m,), levels=levels, ordered=ordered)
 
 CategoricalMatrix(::UndefInitializer, m::Int, n::Int;
                   levels::Union{AbstractVector, Nothing}=nothing,
                   ordered::Bool=false) =
     CategoricalArray(undef, m, n, levels=levels, ordered=ordered)
-CategoricalMatrix{T}(::UndefInitializer, m::Int, n::Int;
-                     levels::Union{AbstractVector, Nothing}=nothing,
-                     ordered::Bool=false) where {T} =
-    CategoricalArray{T}(undef, (m, n), levels=levels, ordered=ordered)
-
 
 ## Constructors from arrays
 
@@ -253,19 +244,11 @@ CategoricalArray(A::AbstractArray{T, N};
                  ordered::Bool=_isordered(A)) where {T, N} =
     CategoricalArray{fixstringtype(T), N}(A, levels=levels, ordered=ordered)
 
-CategoricalVector{T}(A::AbstractVector{S};
-                     levels::Union{AbstractVector, Nothing}=nothing,
-                     ordered=_isordered(A)) where {S, T} =
-    CategoricalArray{T, 1}(A, levels=levels, ordered=ordered)
 CategoricalVector(A::AbstractVector{T};
                   levels::Union{AbstractVector, Nothing}=nothing,
                   ordered::Bool=_isordered(A)) where {T} =
     CategoricalArray{fixstringtype(T), 1}(A, levels=levels, ordered=ordered)
 
-CategoricalMatrix{T}(A::AbstractMatrix{S};
-                     levels::Union{AbstractVector, Nothing}=nothing,
-                     ordered::Bool=_isordered(A)) where {S, T} =
-    CategoricalArray{T, 2}(A, levels=levels, ordered=ordered)
 CategoricalMatrix(A::AbstractMatrix{T};
                   levels::Union{AbstractVector, Nothing}=nothing,
                   ordered::Bool=_isordered(A)) where {T} =
@@ -285,25 +268,16 @@ CategoricalArray(A::CategoricalArray{T, N, R};
                  ordered::Bool=_isordered(A)) where {T, N, R} =
     CategoricalArray{T, N, R}(A, levels=levels, ordered=ordered)
 
-CategoricalVector{T}(A::CategoricalArray{S, 1, R};
-                     levels::Union{AbstractVector, Nothing}=nothing,
-                     ordered::Bool=_isordered(A)) where {S, T, R} =
-    CategoricalArray{T, 1, R}(A, levels=levels, ordered=ordered)
 CategoricalVector(A::CategoricalArray{T, 1, R};
                   levels::Union{AbstractVector, Nothing}=nothing,
                   ordered::Bool=_isordered(A)) where {T, R} =
     CategoricalArray{T, 1, R}(A, levels=levels, ordered=ordered)
 
-CategoricalMatrix{T}(A::CategoricalArray{S, 2, R};
-                     levels::Union{AbstractVector, Nothing}=nothing,
-                     ordered::Bool=_isordered(A)) where {S, T, R} =
-    CategoricalArray{T, 2, R}(A, levels=levels, ordered=ordered)
 CategoricalMatrix(A::CategoricalArray{T, 2, R};
                   levels::Union{AbstractVector, Nothing}=nothing,
                   ordered::Bool=_isordered(A)) where {T, R} =
     CategoricalArray{T, 2, R}(A, levels=levels, ordered=ordered)
 
-
 ## Conversion methods
 
 # From AbstractArray
@@ -365,13 +339,30 @@ function convert(::Type{CategoricalArray{T, N, R}}, A::CategoricalArray{S, N}) w
     refs = convert(Array{R, N}, A.refs)
     CategoricalArray{unwrap_catvaluetype(T), N}(refs, pool)
 end
+
 convert(::Type{CategoricalArray{T, N}}, A::CategoricalArray{S, N, R}) where {S, T, N, R} =
     convert(CategoricalArray{T, N, R}, A)
 convert(::Type{CategoricalArray{T}}, A::CategoricalArray{S, N, R}) where {S, T, N, R} =
     convert(CategoricalArray{T, N, R}, A)
 convert(::Type{CategoricalArray}, A::CategoricalArray{T, N, R}) where {T, N, R} =
     convert(CategoricalArray{T, N, R}, A)
 
+# resolve dispmatch ambiguities with AbstractVector/Matrix
+convert(::Type{CategoricalArray{T, 1}}, A::CategoricalVector{S, R}) where {S, T, R} =
+    convert(CategoricalArray{T, 1, R}, A)
+convert(::Type{CategoricalArray{T}}, A::CategoricalVector{S, R}) where {S, T, R} =
+    convert(CategoricalArray{T, 1, R}, A)
+convert(::Type{CategoricalArray}, A::CategoricalVector{T, R}) where {T, R} =
+    convert(CategoricalArray{T, 1, R}, A)
+
+convert(::Type{CategoricalArray{T, 2}}, A::CategoricalMatrix{S, R}) where {S, T, R} =
+    convert(CategoricalArray{T, 2, R}, A)
+convert(::Type{CategoricalArray{T}}, A::CategoricalMatrix{S, R}) where {S, T, R} =
+    convert(CategoricalArray{T, 2, R}, A)
+convert(::Type{CategoricalArray}, A::CategoricalMatrix{T, R}) where {T, R} =
+    convert(CategoricalArray{T, 2, R}, A)
+
+
 # R<:Integer is needed for this method to be considered more specific
 # than the generic one above (JuliaLang/julia#18443)
 convert(::Type{CategoricalArray{T, N, R}}, A::CategoricalArray{T, N, R}) where {T, N, R<:Integer} = A
@@ -972,3 +963,29 @@ Base.repeat(a::CatArrOrSub{T, N},
 Base.repeat(a::CatArrOrSub{T, N};
             inner = nothing, outer = nothing) where {T, N} =
     CategoricalArray{T, N}(repeat(refs(a), inner=inner, outer=outer), copy(pool(a)))
+
+# JSON3 writing/reading
+StructTypes.StructType(::Type{<:CategoricalVector}) = StructTypes.ArrayType()
+
+StructTypes.construct(::Type{<:CategoricalArray}, array::Vector) =
+    categoricalgeneral(array)
+function categoricalgeneral(array)
+    if eltype(array) >: Nothing
+        categorical(replace(array, nothing=>missing))
+    else
+        categorical(array)
+    end
+end
+
+StructTypes.construct(::Type{<:CategoricalVector{Union{Missing, T}}},
+    array::Vector) where {T, S} = categoricalmissing(T, array)
+StructTypes.construct(::Type{<:CategoricalArray{Union{Missing, T}}},
+    array::Vector) where {T, S} = categoricalmissing(T, array)
+categoricalmissing(T, array) =
+    CategoricalArray{Union{Missing, T}}(replace(array, nothing=>missing))
+
+StructTypes.construct(::Type{<:CategoricalVector{Union{Nothing, T}}},
+    array::Vector) where {T, S} = categoricalnothing(T, array)
+StructTypes.construct(::Type{<:CategoricalArray{Union{Nothing, T}}},
+    array::Vector) where {T, S} = categoricalnothing(T, array)
+categoricalnothing(T, array) = CategoricalArray{Union{Nothing, T}}(array)

src/typedefs.jl

@@ -112,8 +112,8 @@ mutable struct CategoricalArray{T, N, R <: Integer, V, C, U} <: AbstractCategori
         new{T, N, R, V, C, U}(refs, pool)
     end
 end
-const CategoricalVector{T, R, V, C, U} = CategoricalArray{T, 1, V, C, U}
-const CategoricalMatrix{T, R, V, C, U} = CategoricalArray{T, 2, V, C, U}
+const CategoricalVector{T, R <: Integer, V, C, U} = CategoricalArray{T, 1, R, V, C, U}
+const CategoricalMatrix{T, R <: Integer, V, C, U} = CategoricalArray{T, 2, R, V, C, U}
 
 CatArrOrSub{T, N, R} = Union{CategoricalArray{T, N, R},
                              SubArray{<:Any, N, <:CategoricalArray{T, <:Any, R}}} where

src/value.jl

@@ -179,3 +179,9 @@ DataAPI.defaultarray(::Type{CategoricalValue{T, R}}, N) where {T, R} =
   CategoricalArray{T, N, R}
 DataAPI.defaultarray(::Type{Union{CategoricalValue{T, R}, Missing}}, N) where {T, R} =
   CategoricalArray{Union{T, Missing}, N, R}
+
+# define appropriate handlers for JSON3 interface
+StructTypes.StructType(x::CategoricalValue) = StructTypes.StructType(get(x))
+StructTypes.StructType(::Type{<:CategoricalValue{T}}) where {T} = StructTypes.StructType(T)
+StructTypes.numbertype(::Type{<:CategoricalValue{T}}) where {T <: Number} = T
+(::Type{T})(x::CategoricalValue{<:Number}) where {T <: Number} = T(get(x))

test/06_show.jl

@@ -146,4 +146,30 @@ using JSON
     @test typeof(JSON.lower(v)) == typeof(JSON.lower(get(v)))
 end
 
+using JSON3
+using StructTypes
+@testset "JSON3.write" begin
+    v = CategoricalValue(1, CategoricalPool(["a"]))
+    @test JSON3.write(v) === "\"a\""
+
+    v = CategoricalValue(1, CategoricalPool([:a]))
+    @test JSON3.write(v) === "\"a\""
+
+    v = CategoricalValue(1, CategoricalPool([1]))
+    @test JSON3.write(v) === "1"
+    @test StructTypes.numbertype(typeof(v)) === Int
+
+    v = CategoricalValue(1, CategoricalPool([2.0]))
+    @test JSON3.write(v) === "2.0"
+    @test StructTypes.numbertype(typeof(v)) === Float64
+
+    v = CategoricalValue(1, CategoricalPool([BigFloat(3.0,10)]))
+    @test JSON3.write(v) === "3.0"
+    @test StructTypes.numbertype(typeof(v)) === BigFloat
+
+    v = CategoricalValue(2, CategoricalPool([true,false]))
+    @test JSON3.write(v) == "false"
+    @test StructTypes.numbertype(typeof(v)) === Bool
+end
+
 end

test/13_arraycommon.jl

@@ -1827,4 +1827,89 @@ end
     end
 end
 
+using JSON3
+using StructTypes
+
+struct MyCustomTypeMissing
+    id::Vector{Int}
+    var::CategoricalVector{Union{Missing,String}}
+end
+StructTypes.StructType(::Type{<:MyCustomTypeMissing}) = StructTypes.Struct()
+
+struct MyCustomType
+    id::Vector{Int}
+    var::CategoricalVector{String}
+end
+StructTypes.StructType(::Type{<:MyCustomType}) = StructTypes.Struct()
+
+@testset "Reading CategoricalVector objects using JSON3" begin
+    x = CategoricalArray(["x","y","z","y","y","z"])
+    str = JSON3.write(x)
+    readx = JSON3.read(str, CategoricalArray)
+    @test readx == x
+    @test levels(readx) == levels(x)
+    @test readx isa CategoricalArray
+
+    x = CategoricalArray([missing,"y","z","y",missing,"z","x"])
+    str = JSON3.write(x)
+
+    readx = JSON3.read(str, CategoricalVector)
+    @test x ≅ readx
+    @test all(sort(levels(readx)) .== sort(levels(x)))
+    @test readx isa CategoricalVector
+
+    readx = JSON3.read(str, CategoricalArray)
+    @test x ≅ readx
+    @test all(sort(levels(readx)) .== sort(levels(x)))
+    @test readx isa CategoricalVector
+
+    readx = JSON3.read(str, CategoricalArray{Union{Missing,String}})
+    @test x ≅ readx
+    @test levels(readx) == levels(x)
+    @test readx isa CategoricalVector{Union{Missing,String}}
+
+    readx = JSON3.read(str, CategoricalVector{Union{Missing,String}})
+    @test x ≅ readx
+    @test levels(readx) == levels(x)
+    @test readx isa CategoricalVector{Union{Missing,String}}
+
+    readx = JSON3.read(str, CategoricalVector{Union{Nothing,String}})
+    @test all((ismissing(a) && (get(b) isa Nothing)) || a == b for (a,b) in zip(x,readx))
+    @test nothing in levels(readx)
+    @test length(union(setdiff(levels(readx),[nothing]), levels(x))) == length(levels(x))
+    @test readx isa CategoricalVector{Union{Nothing,String}}
+
+    readx = JSON3.read(str, CategoricalArray{Union{Nothing,String}})
+    @test all((ismissing(a) && (get(b) isa Nothing)) || a == b for (a,b) in zip(x,readx))
+    @test nothing in levels(readx)
+    @test length(union(setdiff(levels(readx),[nothing]), levels(x))) == length(levels(x))
+    @test readx isa CategoricalVector{Union{Nothing,String}}
+
+    x = CategoricalArray(["x",nothing,"y","z","y",nothing,"z","x"])
+    str = JSON3.write(x)
+
+    readx = JSON3.read(str, CategoricalArray)
+    @test all(((get(a) isa Nothing) && ismissing(b)) || a == b for (a,b) in zip(x,readx))
+    @test readx isa CategoricalVector
+
+    x = MyCustomType(
+        collect(1:3),
+        CategoricalArray(["x","y","z"])
+    )
+    str = JSON3.write(x)
+    readx = JSON3.read(str, MyCustomType)
+    @test readx.var == x.var
+    @test levels(readx.var) == levels(x.var)
+
+    x = MyCustomTypeMissing(
+        collect(1:3),
+        CategoricalArray(["x","y","z",missing])
+    )
+    str = JSON3.write(x)
+    readx = JSON3.read(str, MyCustomTypeMissing)
+    @test x.var ≅ readx.var
+    @test levels(readx.var) == levels(x.var)
+
+end
+
 end