From e5b286907b7b3ca55d32b41445d250cd6b8a5754 Mon Sep 17 00:00:00 2001 From: Sean Marshallsay Date: Wed, 12 Nov 2014 12:17:33 +0000 Subject: [PATCH 1/3] Allow bytearrays to be stored as BLOBs. Any object that is serialized is now put in a Serialization type then serialized. When sqldeserialize is called on a bytearray, it checks whether the deserialized array is a Serialization and if so returns the object it holds, otherwise returning the array. --- src/SQLite.jl | 32 +++++++++++++++++++++++++++++--- src/UDF.jl | 3 ++- 2 files changed, 31 insertions(+), 4 deletions(-) diff --git a/src/SQLite.jl b/src/SQLite.jl index 08f1dd2..65fc363 100644 --- a/src/SQLite.jl +++ b/src/SQLite.jl @@ -16,6 +16,11 @@ immutable NullType end const NULL = NullType() Base.show(io::IO,::NullType) = print(io,"NULL") +# internal wrapper type to, in-effect, mark something which has been serialized +type Serialization + object +end + type ResultSet colnames values::Vector{Any} @@ -132,13 +137,18 @@ Base.bind(stmt::SQLiteStmt,i::Int,val::Int64) = @CHECK stmt.db sqlite3_ Base.bind(stmt::SQLiteStmt,i::Int,val::NullType) = @CHECK stmt.db sqlite3_bind_null(stmt.handle,i) Base.bind(stmt::SQLiteStmt,i::Int,val::AbstractString) = @CHECK stmt.db sqlite3_bind_text(stmt.handle,i,val) Base.bind(stmt::SQLiteStmt,i::Int,val::UTF16String) = @CHECK stmt.db sqlite3_bind_text16(stmt.handle,i,val) +Base.bind(stmt::SQLiteStmt,i::Int,val::Vector{UInt8}) = @CHECK stmt.db sqlite3_bind_blob(stmt.handle,i,val) # Fallback is BLOB and defaults to serializing the julia value function sqlserialize(x) t = IOBuffer() - serialize(t,x) + # deserialize will sometimes return a random object when called on an array + # which has not been previously serialized, we can use this type to check + # that the array has been serialized + s = Serialization(x) + serialize(t,s) return takebuf_array(t) end -Base.bind(stmt::SQLiteStmt,i::Int,val) = @CHECK stmt.db sqlite3_bind_blob(stmt.handle,i,sqlserialize(val)) +Base.bind(stmt::SQLiteStmt,i::Int,val) = bind(stmt,i,sqlserialize(val)) #TODO: #int sqlite3_bind_zeroblob(sqlite3_stmt*, int, int n); #int sqlite3_bind_value(sqlite3_stmt*, int, const sqlite3_value*); @@ -159,7 +169,23 @@ function execute(db::SQLiteDB,sql::AbstractString) return changes(db) end -sqldeserialize(r) = deserialize(IOBuffer(r)) +function sqldeserialize(r) + # try blocks introduce new scope + local v + # deserialize will sometimes, but not consistently (see comment in + # sqlserialize), throw an error when called on an object which hasn't been + # previously serialized + try + v = deserialize(IOBuffer(r)) + catch + return r + end + if isa(v, Serialization) + return v.object + else + return r + end +end function query(db::SQLiteDB,sql::AbstractString, values=[]) stmt = SQLiteStmt(db,sql) diff --git a/src/UDF.jl b/src/UDF.jl index 669973d..33d62f0 100644 --- a/src/UDF.jl +++ b/src/UDF.jl @@ -30,9 +30,10 @@ sqlreturn(context, val::Int64) = sqlite3_result_int64(context, val) sqlreturn(context, val::Float64) = sqlite3_result_double(context, val) sqlreturn(context, val::UTF16String) = sqlite3_result_text16(context, val) sqlreturn(context, val::AbstractString) = sqlite3_result_text(context, val) -sqlreturn(context, val) = sqlite3_result_blob(context, sqlserialize(val)) +sqlreturn(context, val::Vector{UInt8}) = sqlite3_result_blob(context, sqlserialize(val)) sqlreturn(context, val::Bool) = sqlreturn(context, int(val)) +sqlreturn(context, val) = sqlreturn(context, sqlserialize(val)) # Internal method for generating an SQLite scalar function from # a Julia function name From 45c02cf34d8920ab844662067a4a4f0b9d6d4c6a Mon Sep 17 00:00:00 2001 From: Sean Marshallsay Date: Wed, 12 Nov 2014 13:40:17 +0000 Subject: [PATCH 2/3] Test sqlserialize and sqldeserialize more thoroughly. --- test/runtests.jl | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/test/runtests.jl b/test/runtests.jl index 78dd6b6..023d0ab 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -114,6 +114,34 @@ r = query(db, "SELECT * FROM temp WHERE AlbumId = 0") @test r == ResultSet(Any["AlbumId", "Title", "ArtistId"], Any[Any[0], Any["Test Album"], Any[0]]) drop(db, "temp") +binddb = SQLiteDB() +query(binddb, "CREATE TABLE temp (n NULL, i6 INT, f REAL, s TEXT, a BLOB)") +query(binddb, "INSERT INTO temp VALUES (?1, ?2, ?3, ?4, ?5)", Any[NULL, int64(6), 6.4, "some text", b"bytearray"]) +r = query(binddb, "SELECT * FROM temp") +for (v, t) in zip(r.values, [SQLite.NullType, Int64, Float64, AbstractString, Vector{UInt8}]) + @test isa(v[1], t) +end +query(binddb, "CREATE TABLE blobtest (a BLOB, b BLOB)") +query(binddb, "INSERT INTO blobtest VALUES (?1, ?2)", Any[b"a", b"b"]) +query(binddb, "INSERT INTO blobtest VALUES (?1, ?2)", Any[b"a", BigInt(2)]) +type Point{T} + x::T + y::T +end +==(a::Point, b::Point) = a.x == b.x && a.y == b.y +p1 = Point(1, 2) +p2 = Point(1.3, 2.4) +query(binddb, "INSERT INTO blobtest VALUES (?1, ?2)", Any[b"a", p1]) +query(binddb, "INSERT INTO blobtest VALUES (?1, ?2)", Any[b"a", p2]) +r = query(binddb, "SELECT * FROM blobtest") +for v in r.values[1] + @test v == b"a" +end +for (v1, v2) in zip(r.values[2], Any[b"b", BigInt(2), p1, p2]) + @test v1 == v2 +end +close(binddb) + # I can't be arsed to create a new one using old dictionary syntax if VERSION > v"0.4.0-" query(db,"CREATE TABLE temp AS SELECT * FROM Album") From f5b4876b2c26c88c75557517c1a1c500f598ea9f Mon Sep 17 00:00:00 2001 From: Sean Marshallsay Date: Sun, 16 Nov 2014 12:30:53 +0000 Subject: [PATCH 3/3] Fix serialization in sqlreturn and more tests. sqlreturn(::Any, ::Vector{UInt8}) was serializing values so any values passed to sqlreturn(::Any, ::Any) were being serialized twice. The net result of this being that they were returned as bytearrays. --- src/UDF.jl | 2 +- test/runtests.jl | 8 ++++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/src/UDF.jl b/src/UDF.jl index 33d62f0..95ac2eb 100644 --- a/src/UDF.jl +++ b/src/UDF.jl @@ -30,7 +30,7 @@ sqlreturn(context, val::Int64) = sqlite3_result_int64(context, val) sqlreturn(context, val::Float64) = sqlite3_result_double(context, val) sqlreturn(context, val::UTF16String) = sqlite3_result_text16(context, val) sqlreturn(context, val::AbstractString) = sqlite3_result_text(context, val) -sqlreturn(context, val::Vector{UInt8}) = sqlite3_result_blob(context, sqlserialize(val)) +sqlreturn(context, val::Vector{UInt8}) = sqlite3_result_blob(context, val) sqlreturn(context, val::Bool) = sqlreturn(context, int(val)) sqlreturn(context, val) = sqlreturn(context, sqlserialize(val)) diff --git a/test/runtests.jl b/test/runtests.jl index 023d0ab..55aafbb 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -189,6 +189,14 @@ SQLite.register(db, hypot; nargs=2, name="hypotenuse") v = query(db, "select hypotenuse(Milliseconds,bytes) from track limit 5") @test [int(i) for i in v[1]] == [11175621,5521062,3997652,4339106,6301714] +SQLite.@register db str2arr(s) = convert(Array{UInt8}, s) +r = query(db, "SELECT str2arr(LastName) FROM Employee LIMIT 2") +@test r[1] == Any[UInt8[0x41,0x64,0x61,0x6d,0x73],UInt8[0x45,0x64,0x77,0x61,0x72,0x64,0x73]] + +SQLite.@register db big +r = query(db, "SELECT big(5)") +@test r[1][1] == big(5) + @test size(tables(db)) == (11,1) close(db)