Merge pull request #1471 from JuliaData/jq/1.0

Updates to run on 1.0

.travis.yml

@@ -20,8 +20,8 @@ before_script:
 
 script:
   - if [[ -a .git/shallow ]]; then git fetch --unshallow; fi
-  - julia --check-bounds=yes -e 'Pkg.clone(pwd()); Pkg.build("DataFrames"); Pkg.test("DataFrames"; coverage=true)'
+  - julia --check-bounds=yes -e 'using Pkg; Pkg.clone(pwd()); Pkg.build("DataFrames"); Pkg.test("DataFrames"; coverage=true)'
 
 after_success:
-  - julia -e 'cd(Pkg.dir("DataFrames")); Pkg.add("Documenter"); Pkg.add("Query"); Pkg.add("CSV"); include(joinpath("docs", "make.jl"))'
-  - julia -e 'cd(Pkg.dir("DataFrames")); Pkg.add("Coverage"); using Coverage; Coveralls.submit(Coveralls.process_folder())'
+  - julia -e 'using Pkg; cd(Pkg.dir("DataFrames")); Pkg.add("Documenter"); Pkg.add("Query"); Pkg.add("CSV"); include(joinpath("docs", "make.jl"))'
+  - julia -e 'using Pkg; cd(Pkg.dir("DataFrames")); Pkg.add("Coverage"); using Coverage; Coveralls.submit(Coveralls.process_folder())'

src/abstractdataframe/abstractdataframe.jl

@@ -82,9 +82,10 @@ abstract type AbstractDataFrame end
 struct Cols{T <: AbstractDataFrame} <: AbstractVector{Any}
     df::T
 end
-Base.start(::Cols) = 1
-Base.done(itr::Cols, st) = st > length(itr.df)
-Base.next(itr::Cols, st) = (itr.df[st], st + 1)
+function Base.iterate(c::Cols, st=1)
+    st > length(itr.df) && return nothing
+    return (itr.df[st], st + 1)
+end
 Base.length(itr::Cols) = length(itr.df)
 Base.size(itr::Cols, ix) = ix==1 ? length(itr) : throw(ArgumentError("Incorrect dimension"))
 Base.size(itr::Cols) = (length(itr.df),)

src/abstractdataframe/iteration.jl

@@ -19,13 +19,9 @@ which is a view that acts like a one-row DataFrame.
 eachrow(df::AbstractDataFrame) = DFRowIterator(df)
 
 function Base.iterate(itr::DFRowIterator, i=1)
-    if i > size(itr.df, 1)
-        nothing
-    else
-        (DataFrameRow(itr.df, i), i + 1)
-    end
+    i > size(itr.df, 1) && return nothing
+    return (DataFrameRow(itr.df, i), i + 1)
 end
-
 Base.size(itr::DFRowIterator) = (size(itr.df, 1), )
 Base.length(itr::DFRowIterator) = size(itr.df, 1)
 Base.getindex(itr::DFRowIterator, i::Any) = DataFrameRow(itr.df, i)
@@ -37,14 +33,10 @@ struct DFColumnIterator{T <: AbstractDataFrame}
 end
 eachcol(df::AbstractDataFrame) = DFColumnIterator(df)
 
-function Base.iterate(itr::DFColumnIterator, i=1)
-    if i > size(itr.df, 2)
-        nothing
-    else
-        ((_names(itr.df)[i], itr.df[i]), i + 1)
-    end
+function Base.iterate(itr::DFColumnIterator, j=1)
+    j > size(itr.df, 2) && return nothing
+    return ((_names(itr.df)[j], itr.df[j]), j + 1)
 end
-
 Base.size(itr::DFColumnIterator) = (size(itr.df, 2), )
 Base.length(itr::DFColumnIterator) = size(itr.df, 2)
 Base.getindex(itr::DFColumnIterator, j::Any) = itr.df[:, j]

src/abstractdataframe/join.jl

@@ -173,6 +173,8 @@ function update_row_maps!(left_table::AbstractDataFrame,
     end
 end
 
+adjustrange(i, rng) = UnitRange(rng.start + i, rng.stop + i)
+
 # map the row indices of the left and right joined tables
 # to the indices of rows in the resulting table
 # returns the 4-tuple of row indices maps for
@@ -202,9 +204,9 @@ function update_row_maps!(left_table::AbstractDataFrame,
     if map_rightonly
         rightonly_orig_ixs = findall(rightonly_mask)
         rightonly_ixs = RowIndexMap(rightonly_orig_ixs,
-                                    collect(length(right_ixs.orig) +
-                                            (leftonly_ixs === nothing ? 0 : length(leftonly_ixs)) +
-                                            (1:length(rightonly_orig_ixs))))
+                                    collect(adjustrange(length(right_ixs.orig) +
+                                            (leftonly_ixs === nothing ? 0 : length(leftonly_ixs)),
+                                            1:length(rightonly_orig_ixs))))
     else
         rightonly_ixs = nothing
     end

src/abstractdataframe/show.jl

@@ -1,3 +1,9 @@
+@static if isdefined(Base, :showall)
+    import Base: showall
+else
+    export showall
+end
+
 #' @exported
 #' @description
 #'
@@ -508,7 +514,7 @@ end
 #'
 #' df = DataFrame(A = 1:3, B = ["x", "y", "z"])
 #' showall(stdout, df, false, :Row, true)
-function Base.showall(io::IO,
+function showall(io::IO,
                       df::AbstractDataFrame,
                       allcols::Bool = true,
                       rowlabel::Symbol = :Row,
@@ -545,8 +551,10 @@ end
 #'
 #' df = DataFrame(A = 1:3, B = ["x", "y", "z"])
 #' showall(df, true)
-function Base.showall(df::AbstractDataFrame,
+function showall(df::AbstractDataFrame,
                       allcols::Bool = true) # -> Nothing
     showall(stdout, df, allcols)
     return
 end
+
+showall(io::IO, dfvec::AbstractVector{T}) where {T <: AbstractDataFrame} = foreach(df->showall(io, df), dfvec)

src/abstractdataframe/sort.jl

@@ -229,7 +229,7 @@ Sort.defalg(df::AbstractDataFrame) = size(df, 1) < 8192 ? Sort.MergeSort : Sorti
 function Sort.defalg(df::AbstractDataFrame, ::Type{T}, o::Ordering) where T<:Real
     # If we're sorting a single numerical column in forward or reverse,
     # RadixSort will generally be the fastest stable sort
-    if isbits(T) && sizeof(T) <= 8 && (o==Order.Forward || o==Order.Reverse)
+    if isbitstype(T) && sizeof(T) <= 8 && (o==Order.Forward || o==Order.Reverse)
         SortingAlgorithms.RadixSort
     else
         Sort.defalg(df)

src/dataframe/dataframe.jl

@@ -121,7 +121,7 @@ mutable struct DataFrame <: AbstractDataFrame
 end
 
 function DataFrame(pairs::Pair{Symbol,<:Any}...; makeunique::Bool=false)::DataFrame
-    colnames = Symbol[k for (k,v) in pairs]
+    colnames = [Symbol(k) for (k,v) in pairs]
     columns = Any[v for (k,v) in pairs]
     DataFrame(columns, Index(colnames, makeunique=makeunique))
 end
@@ -133,7 +133,7 @@ function DataFrame(d::AbstractDict)
     else
         colnames = keys(d)
     end
-    colindex = Index(Symbol[k for k in colnames])
+    colindex = Index([Symbol(k) for k in colnames])
     columns = Any[d[c] for c in colnames]
     DataFrame(columns, colindex)
 end
@@ -1104,5 +1104,6 @@ function permutecols!(df::DataFrame, p::AbstractVector)
 end
 
 function permutecols!(df::DataFrame, p::AbstractVector{Symbol})
-    permutecols!(df, getindex.(index(df).lookup, p))
+    lu = index(df).lookup
+    permutecols!(df, [lu[x] for x in p])
 end

src/dataframerow/dataframerow.jl

@@ -43,11 +43,10 @@ Compat.lastindex(r::DataFrameRow) = size(parent(r), 2)
 
 Base.collect(r::DataFrameRow) = Tuple{Symbol, Any}[x for x in r]
 
-Base.start(r::DataFrameRow) = 1
-
-Base.next(r::DataFrameRow, s) = ((_names(r)[s], r[s]), s + 1)
-
-Base.done(r::DataFrameRow, s) = s > length(r)
+function Base.iterate(r::DataFrameRow, st=1)
+    st > length(r) && return nothing
+    return ((_names(r)[st], r[st]), st + 1)
+end
 
 Base.convert(::Type{Array}, r::DataFrameRow) = convert(Array, parent(r)[row(r),:])
 

src/deprecated.jl

@@ -21,7 +21,6 @@ import Base: keys, values, insert!
 
 @deprecate sub(df::AbstractDataFrame, rows) view(df, rows)
 
-
 ## write.table
 using CodecZlib, TranscodingStreams
 

src/groupeddataframe/show.jl

@@ -12,7 +12,7 @@ function Base.show(io::IO, gd::GroupedDataFrame)
     end
 end
 
-function Base.showall(io::IO, gd::GroupedDataFrame)
+function showall(io::IO, gd::GroupedDataFrame)
     N = length(gd)
     println(io, "$(typeof(gd))  $N groups with keys: $(gd.cols)")
     for i = 1:N

src/other/utils.jl

@@ -1,10 +1,10 @@
 import Base: isidentifier, is_id_start_char, is_id_char
 
-const RESERVED_WORDS = Set(["begin", "while", "if", "for", "try",
-    "return", "break", "continue", "function", "macro", "quote", "let",
-    "local", "global", "const", "type",
-    "immutable", "do", "module", "baremodule", "using", "import", "struct",
-    "export", "importall", "end", "else", "elseif", "catch", "finally"])
+const RESERVED_WORDS = Set(["local", "global", "export", "let",
+    "for", "struct", "while", "const", "continue", "import",
+    "function", "if", "else", "try", "begin", "break", "catch",
+    "return", "using", "baremodule", "macro", "finally",
+    "module", "elseif", "end", "quote", "do"])
 
 function identifier(s::AbstractString)
     s = Unicode.normalize(s)

test/conversions.jl

@@ -49,23 +49,23 @@ module TestConversions
 
     df = convert(DataFrame, di)
     @test isa(df, DataFrame)
-    @test names(df) == Symbol[x for x in sort(collect(keys(di)))]
+    @test names(df) == [Symbol(x) for x in sort(collect(keys(di)))]
     @test df[:a] == a
     @test df[:b] == b
     @test df[:c] == c
 
     od = OrderedDict("c"=>c, "a"=>a, "b"=>b)
     df = convert(DataFrame,od)
     @test isa(df, DataFrame)
-    @test names(df) == Symbol[x for x in keys(od)]
+    @test names(df) == [Symbol(x) for x in keys(od)]
     @test df[:a] == a
     @test df[:b] == b
     @test df[:c] == c
 
     sd = SortedDict("c"=>c, "a"=>a, "b"=>b)
     df = convert(DataFrame,sd)
     @test isa(df, DataFrame)
-    @test names(df) == Symbol[x for x in keys(sd)]
+    @test names(df) == [Symbol(x) for x in keys(sd)]
     @test df[:a] == a
     @test df[:b] == b
     @test df[:c] == c

test/data.jl

@@ -75,7 +75,7 @@ module TestData
         #test_group("Associative")
 
         #test_group("DataFrame")
-        srand(1)
+        Random.seed!(1)
         N = 20
         d1 = Vector{Union{Int64, Missing}}(rand(1:2, N))
         d2 = CategoricalArray(["A", "B", missing])[rand(1:3, N)]
@@ -221,7 +221,7 @@ module TestData
     end
 
     @testset "merge" begin
-        srand(1)
+        Random.seed!(1)
         df1 = DataFrame(a = shuffle!(Vector{Union{Int, Missing}}(1:10)),
                         b = rand(Union{Symbol, Missing}[:A,:B], 10),
                         v1 = Vector{Union{Float64, Missing}}(randn(10)))
@@ -277,7 +277,7 @@ module TestData
         @test m2[:A] ≅ ["a", "b", "a", missing, "c"]
     end
 
-    srand(1)
+    Random.seed!(1)
     df1 = DataFrame(
         a = rand(Union{Symbol, Missing}[:x,:y], 10),
         b = rand(Union{Symbol, Missing}[:A,:B], 10),
@@ -297,7 +297,7 @@ module TestData
     @test ismissing(m2[10,:v2])
     @test m2[:a] ≅ [:x, :x, :y, :y, :y, :x, :x, :y, :x, :y, missing, :y]
 
-    srand(1)
+    Random.seed!(1)
     function spltdf(d)
         d[:x1] = map(x -> x[1], d[:a])
         d[:x2] = map(x -> x[2], d[:a])

test/dataframe.jl

@@ -179,7 +179,7 @@ module TestDataFrame
         @test size(df, 2) == 5
         @test typeof(df[:, 1]) == Vector{Float64}
 
-        df = convert(DataFrame, Matrix{Float64}(I, 10, 5))
+        df = convert(DataFrame, Matrix{Float64}(undef, 10, 5))
         @test size(df, 1) == 10
         @test size(df, 2) == 5
         @test typeof(df[:, 1]) == Vector{Float64}
@@ -190,7 +190,7 @@ module TestDataFrame
 
         # This assignment was missing before
         df = DataFrame(Column = [:A])
-        df[1, :Column] = "Testing"
+        df[1, :Column] = :Testing
 
         # zero-row DataFrame and subDataFrame test
         df = DataFrame(x=[], y=[])
@@ -642,14 +642,14 @@ module TestDataFrame
         df = DataFrame(Any[CategoricalArray(1:10),
                            CategoricalArray(string.('a':'j'))])
         allowmissing!(df)
-        @test all(issubtype.(typeof.(columns(df)), CategoricalVector))
+        @test all(x->x <: CategoricalVector, typeof.(columns(df)))
         @test eltypes(df)[1] <: Union{CategoricalValue{Int}, Missing}
         @test eltypes(df)[2] <: Union{CategoricalString, Missing}
         df[1,2] = missing
         @test_throws MissingException disallowmissing!(df)
         df[1,2] = "a"
         disallowmissing!(df)
-        @test all(issubtype.(typeof.(columns(df)), CategoricalVector))
+        @test all(x->x <: CategoricalVector, typeof.(columns(df)))
         @test eltypes(df)[1] <: CategoricalValue{Int}
         @test eltypes(df)[2] <: CategoricalString
     end

test/grouping.jl

@@ -2,7 +2,7 @@ module TestGrouping
     using Test, DataFrames, Random
     const ≅ = isequal
 
-    srand(1)
+    Random.seed!(1)
     df = DataFrame(a = repeat(Union{Int, Missing}[1, 2, 3, 4], outer=[2]),
                    b = repeat(Union{Int, Missing}[2, 1], outer=[4]),
                    c = Vector{Union{Float64, Missing}}(randn(8)))

test/io.jl

@@ -1,28 +1,29 @@
 module TestIO
     using Test, DataFrames, CategoricalArrays, Dates
-    using LaTeXStrings
+    # commented out for 1.0 release because LaTeXStrings can't load; re-enable once it has updated for 1.0
+    # using LaTeXStrings
 
-    # Test LaTeX export
-    @testset "LaTeX export" begin
-        df = DataFrame(A = 1:4,
-                    B = ["\$10.0", "M&F", "A~B", "\\alpha"],
-                    C = [L"\alpha", L"\beta", L"\gamma", L"\sum_{i=1}^n \delta_i"],
-                    D = [1.0, 2.0, missing, 3.0],
-                    E = CategoricalArray(["a", missing, "c", "d"]),
-                    F = Vector{String}(undef, 4)
-                    )
-        str = """
-            \\begin{tabular}{r|cccccc}
-            \t& A & B & C & D & E & F\\\\
-            \t\\hline
-            \t1 & 1 & \\\$10.0 & \$\\alpha\$ & 1.0 & a & \\#undef \\\\
-            \t2 & 2 & M\\&F & \$\\beta\$ & 2.0 &  & \\#undef \\\\
-            \t3 & 3 & A\\textasciitilde{}B & \$\\gamma\$ &  & c & \\#undef \\\\
-            \t4 & 4 & \\textbackslash{}\\textbackslash{}alpha & \$\\sum_{i=1}^n \\delta_i\$ & 3.0 & d & \\#undef \\\\
-            \\end{tabular}
-            """
-        @test repr(MIME("text/latex"), df) == str
-    end
+    # # Test LaTeX export
+    # @testset "LaTeX export" begin
+    #     df = DataFrame(A = 1:4,
+    #                 B = ["\$10.0", "M&F", "A~B", "\\alpha"],
+    #                 C = [L"\alpha", L"\beta", L"\gamma", L"\sum_{i=1}^n \delta_i"],
+    #                 D = [1.0, 2.0, missing, 3.0],
+    #                 E = CategoricalArray(["a", missing, "c", "d"]),
+    #                 F = Vector{String}(undef, 4)
+    #                 )
+    #     str = """
+    #         \\begin{tabular}{r|cccccc}
+    #         \t& A & B & C & D & E & F\\\\
+    #         \t\\hline
+    #         \t1 & 1 & \\\$10.0 & \$\\alpha\$ & 1.0 & a & \\#undef \\\\
+    #         \t2 & 2 & M\\&F & \$\\beta\$ & 2.0 &  & \\#undef \\\\
+    #         \t3 & 3 & A\\textasciitilde{}B & \$\\gamma\$ &  & c & \\#undef \\\\
+    #         \t4 & 4 & \\textbackslash{}\\textbackslash{}alpha & \$\\sum_{i=1}^n \\delta_i\$ & 3.0 & d & \\#undef \\\\
+    #         \\end{tabular}
+    #         """
+    #     @test repr(MIME("text/latex"), df) == str
+    # end
 
     #Test HTML output for IJulia and similar
     @testset "HTML output" begin

test/runtests.jl

@@ -6,7 +6,7 @@ fatalerrors = length(ARGS) > 0 && ARGS[1] == "-f"
 quiet = length(ARGS) > 0 && ARGS[1] == "-q"
 anyerrors = false
 
-using Test, DataFrames
+using Test, Random, DataFrames
 
 my_tests = ["utils.jl",
             "cat.jl",

test/show.jl

@@ -21,7 +21,7 @@ module TestShow
         @test str == refstr
     end
 
-    srand(1)
+    Random.seed!(1)
     df_big = DataFrame(rand(25,5))
 
     io = IOContext(IOBuffer(), :displaysize=>(10,40))

test/sort.jl

@@ -66,7 +66,7 @@ module TestSort
 
     @test_throws ArgumentError sort(x, by=:x)
 
-    srand(1)
+    Random.seed!(1)
     # here there will be probably no ties
     df_rand1 = DataFrame(rand(100, 4))
     # but here we know we will have ties