Fix column type detection with missing values

Project.toml

@@ -18,7 +18,7 @@ XLSX = "fdbf4ff8-1666-58a4-91e7-1b58723a45e0"
 
 [compat]
 DataFrames = "≥ 0.14.1"
-DataValues = "≥ 0.4.4"
+DataValues = "≥ 0.4.11"
 ExcelReaders = "≥ 0.10.2"
 FileIO = "≥ 1.0.0"
 IterableTables = "≥ 0.8.3"

src/ExcelFiles.jl

@@ -87,7 +87,7 @@ function _readxl(file::ExcelReaders.ExcelFile, sheetname::AbstractString, startr
         end
 
         if type_of_el <: DataValue
-            columns[i] = convert(DataValueArray, vals)
+            columns[i] = convert(DataValueArray{eltype(type_of_el)}, vals)
 
             # TODO Check wether this hack is correct
             for (j,v) in enumerate(columns[i])

test/runtests.jl

@@ -16,7 +16,7 @@ efile = load(filename, "Sheet1")
 
 @test sprint((stream,data)->show(stream, "text/html", data), efile) == "<table><thead><tr><th>Some Float64s</th><th>Some Strings</th><th>Some Bools</th><th>Mixed column</th><th>Mixed with NA</th><th>Float64 with NA</th><th>String with NA</th><th>Bool with NA</th><th>Some dates</th><th>Dates with NA</th><th>Some errors</th><th>Errors with NA</th><th>Column with NULL and then mixed</th></tr></thead><tbody><tr><td>1.0</td><td>&quot;A&quot;</td><td>true</td><td>2.0</td><td>9.0</td><td>3.0</td><td>&quot;FF&quot;</td><td>#NA</td><td>2015-03-03T00:00:00</td><td>1965-04-03T00:00:00</td><td>#DIV/0&#33;</td><td>#DIV/0&#33;</td><td>#NA</td></tr><tr><td>1.5</td><td>&quot;BB&quot;</td><td>false</td><td>&quot;EEEEE&quot;</td><td>&quot;III&quot;</td><td>#NA</td><td>#NA</td><td>true</td><td>2015-02-04T10:14:00</td><td>1950-08-09T18:40:00</td><td>#N/A</td><td>#N/A</td><td>3.4</td></tr><tr><td>2.0</td><td>&quot;CCC&quot;</td><td>false</td><td>false</td><td>#NA</td><td>3.5</td><td>&quot;GGG&quot;</td><td>#NA</td><td>1988-04-09T00:00:00</td><td>19:00:00</td><td>#REF&#33;</td><td>#NAME?</td><td>&quot;HKEJW&quot;</td></tr><tr><td>2.5</td><td>&quot;DDDD&quot;</td><td>true</td><td>1.5</td><td>true</td><td>4.0</td><td>&quot;HHHH&quot;</td><td>false</td><td>15:02:00</td><td>#NA</td><td>#NAME?</td><td>#NA</td><td>#NA</td></tr></tbody></table>"
 
-@test sprint((stream,data)->show(stream, "application/vnd.dataresource+json", data), efile) == "{\"schema\":{\"fields\":[{\"name\":\"Some Float64s\",\"type\":\"number\"},{\"name\":\"Some Strings\",\"type\":\"string\"},{\"name\":\"Some Bools\",\"type\":\"boolean\"},{\"name\":\"Mixed column\",\"type\":\"string\"},{\"name\":\"Mixed with NA\",\"type\":\"string\"},{\"name\":\"Float64 with NA\",\"type\":\"string\"},{\"name\":\"String with NA\",\"type\":\"string\"},{\"name\":\"Bool with NA\",\"type\":\"string\"},{\"name\":\"Some dates\",\"type\":\"string\"},{\"name\":\"Dates with NA\",\"type\":\"string\"},{\"name\":\"Some errors\",\"type\":\"string\"},{\"name\":\"Errors with NA\",\"type\":\"string\"},{\"name\":\"Column with NULL and then mixed\",\"type\":\"string\"}]},\"data\":[{\"Some Float64s\":1.0,\"Some Strings\":\"A\",\"Some Bools\":true,\"Mixed column\":2.0,\"Mixed with NA\":9.0,\"Float64 with NA\":3.0,\"String with NA\":\"FF\",\"Bool with NA\":null,\"Some dates\":\"2015-03-03T00:00:00\",\"Dates with NA\":\"1965-04-03T00:00:00\",\"Some errors\":{\"errorcode\":7},\"Errors with NA\":{\"errorcode\":7},\"Column with NULL and then mixed\":null},{\"Some Float64s\":1.5,\"Some Strings\":\"BB\",\"Some Bools\":false,\"Mixed column\":\"EEEEE\",\"Mixed with NA\":\"III\",\"Float64 with NA\":null,\"String with NA\":null,\"Bool with NA\":true,\"Some dates\":\"2015-02-04T10:14:00\",\"Dates with NA\":\"1950-08-09T18:40:00\",\"Some errors\":{\"errorcode\":42},\"Errors with NA\":{\"errorcode\":42},\"Column with NULL and then mixed\":3.4},{\"Some Float64s\":2.0,\"Some Strings\":\"CCC\",\"Some Bools\":false,\"Mixed column\":false,\"Mixed with NA\":null,\"Float64 with NA\":3.5,\"String with NA\":\"GGG\",\"Bool with NA\":null,\"Some dates\":\"1988-04-09T00:00:00\",\"Dates with NA\":\"19:00:00\",\"Some errors\":{\"errorcode\":23},\"Errors with NA\":{\"errorcode\":29},\"Column with NULL and then mixed\":\"HKEJW\"},{\"Some Float64s\":2.5,\"Some Strings\":\"DDDD\",\"Some Bools\":true,\"Mixed column\":1.5,\"Mixed with NA\":true,\"Float64 with NA\":4.0,\"String with NA\":\"HHHH\",\"Bool with NA\":false,\"Some dates\":\"15:02:00\",\"Dates with NA\":null,\"Some errors\":{\"errorcode\":29},\"Errors with NA\":null,\"Column with NULL and then mixed\":null}]}"
+@test sprint((stream,data)->show(stream, "application/vnd.dataresource+json", data), efile) == "{\"schema\":{\"fields\":[{\"name\":\"Some Float64s\",\"type\":\"number\"},{\"name\":\"Some Strings\",\"type\":\"string\"},{\"name\":\"Some Bools\",\"type\":\"boolean\"},{\"name\":\"Mixed column\",\"type\":\"string\"},{\"name\":\"Mixed with NA\",\"type\":\"string\"},{\"name\":\"Float64 with NA\",\"type\":\"number\"},{\"name\":\"String with NA\",\"type\":\"string\"},{\"name\":\"Bool with NA\",\"type\":\"boolean\"},{\"name\":\"Some dates\",\"type\":\"string\"},{\"name\":\"Dates with NA\",\"type\":\"string\"},{\"name\":\"Some errors\",\"type\":\"string\"},{\"name\":\"Errors with NA\",\"type\":\"string\"},{\"name\":\"Column with NULL and then mixed\",\"type\":\"string\"}]},\"data\":[{\"Some Float64s\":1.0,\"Some Strings\":\"A\",\"Some Bools\":true,\"Mixed column\":2.0,\"Mixed with NA\":9.0,\"Float64 with NA\":3.0,\"String with NA\":\"FF\",\"Bool with NA\":null,\"Some dates\":\"2015-03-03T00:00:00\",\"Dates with NA\":\"1965-04-03T00:00:00\",\"Some errors\":{\"errorcode\":7},\"Errors with NA\":{\"errorcode\":7},\"Column with NULL and then mixed\":null},{\"Some Float64s\":1.5,\"Some Strings\":\"BB\",\"Some Bools\":false,\"Mixed column\":\"EEEEE\",\"Mixed with NA\":\"III\",\"Float64 with NA\":null,\"String with NA\":null,\"Bool with NA\":true,\"Some dates\":\"2015-02-04T10:14:00\",\"Dates with NA\":\"1950-08-09T18:40:00\",\"Some errors\":{\"errorcode\":42},\"Errors with NA\":{\"errorcode\":42},\"Column with NULL and then mixed\":3.4},{\"Some Float64s\":2.0,\"Some Strings\":\"CCC\",\"Some Bools\":false,\"Mixed column\":false,\"Mixed with NA\":null,\"Float64 with NA\":3.5,\"String with NA\":\"GGG\",\"Bool with NA\":null,\"Some dates\":\"1988-04-09T00:00:00\",\"Dates with NA\":\"19:00:00\",\"Some errors\":{\"errorcode\":23},\"Errors with NA\":{\"errorcode\":29},\"Column with NULL and then mixed\":\"HKEJW\"},{\"Some Float64s\":2.5,\"Some Strings\":\"DDDD\",\"Some Bools\":true,\"Mixed column\":1.5,\"Mixed with NA\":true,\"Float64 with NA\":4.0,\"String with NA\":\"HHHH\",\"Bool with NA\":false,\"Some dates\":\"15:02:00\",\"Dates with NA\":null,\"Some errors\":{\"errorcode\":29},\"Errors with NA\":null,\"Column with NULL and then mixed\":null}]}"
 
 @test sprint(show, efile) == "4x13 Excel file\nSome Float64s │ Some Strings │ Some Bools │ Mixed column │ Mixed with NA\n──────────────┼──────────────┼────────────┼──────────────┼──────────────\n1.0           │ A            │ true       │ 2.0          │ 9.0          \n1.5           │ BB           │ false      │ \"EEEEE\"      │ \"III\"        \n2.0           │ CCC          │ false      │ false        │ #NA          \n2.5           │ DDDD         │ true       │ 1.5          │ true         \n... with 8 more columns: Float64 with NA, String with NA, Bool with NA, Some dates, Dates with NA, Some errors, Errors with NA, Column with NULL and then mixed"