From 2fad00e16a6a0e5546f777ec231d5974f214d763 Mon Sep 17 00:00:00 2001 From: Brice Lambson Date: Thu, 27 Feb 2020 13:28:47 -0800 Subject: [PATCH] Microsoft.Data.SQLite: Bring back the SqliteBlob optimizations This reverts commits 2fdcc8d560c9619153857f1d31556bbd2f1228c0 and b416e9d28babc5f65af8c0d38b75500f671891e3. Fixes #13987 --- .../SqliteDataReader.cs | 10 ++ .../SqliteDataRecord.cs | 118 +++++++++------- .../SqliteDataReaderTest.cs | 127 ++++++++++++++++++ 3 files changed, 203 insertions(+), 52 deletions(-) diff --git a/src/Microsoft.Data.Sqlite.Core/SqliteDataReader.cs b/src/Microsoft.Data.Sqlite.Core/SqliteDataReader.cs index bd4fbc4a875..93fe3834ddc 100644 --- a/src/Microsoft.Data.Sqlite.Core/SqliteDataReader.cs +++ b/src/Microsoft.Data.Sqlite.Core/SqliteDataReader.cs @@ -544,6 +544,16 @@ public override Stream GetStream(int ordinal) ? throw new InvalidOperationException(Resources.NoData) : _record.GetStream(ordinal); + /// + /// Retrieves data as a . + /// + /// The zero-based column ordinal. + /// The returned object. + public override TextReader GetTextReader(int ordinal) + => IsDBNull(ordinal) + ? (TextReader)new StringReader(string.Empty) + : new StreamReader(GetStream(ordinal), Encoding.UTF8); + /// /// Gets the value of the specified column. /// diff --git a/src/Microsoft.Data.Sqlite.Core/SqliteDataRecord.cs b/src/Microsoft.Data.Sqlite.Core/SqliteDataRecord.cs index 44d3121eb6b..a1f4cae9902 100644 --- a/src/Microsoft.Data.Sqlite.Core/SqliteDataRecord.cs +++ b/src/Microsoft.Data.Sqlite.Core/SqliteDataRecord.cs @@ -5,6 +5,7 @@ using System.Diagnostics; using System.IO; using System.Linq; +using System.Text; using Microsoft.Data.Sqlite.Properties; using SQLitePCL; using static SQLitePCL.raw; @@ -17,6 +18,7 @@ internal class SqliteDataRecord : SqliteValueReader, IDisposable private readonly byte[][] _blobCache; private readonly int?[] _typeCache; private bool _stepped; + private int? _rowidOrdinal; public SqliteDataRecord(sqlite3_stmt stmt, bool hasRows, SqliteConnection connection) { @@ -196,26 +198,32 @@ public static Type GetFieldType(string type) public virtual long GetBytes(int ordinal, long dataOffset, byte[] buffer, int bufferOffset, int length) { - var blob = GetCachedBlob(ordinal); + using var stream = GetStream(ordinal); - long bytesToRead = blob.Length - dataOffset; - if (buffer != null) + if (buffer == null) { - bytesToRead = Math.Min(bytesToRead, length); - Array.Copy(blob, dataOffset, buffer, bufferOffset, bytesToRead); + return stream.Length - dataOffset; } - return bytesToRead; + stream.Position = dataOffset; + + return stream.Read(buffer, bufferOffset, length); } public virtual long GetChars(int ordinal, long dataOffset, char[] buffer, int bufferOffset, int length) { - var text = GetString(ordinal); + using var reader = new StreamReader(GetStream(ordinal), Encoding.UTF8); - int charsToRead = text.Length - (int)dataOffset; - charsToRead = Math.Min(charsToRead, length); - text.CopyTo((int)dataOffset, buffer, bufferOffset, charsToRead); - return charsToRead; + for (var position = 0; position < dataOffset; position++) + { + if (reader.Read() == -1) + { + // NB: Message is provided by the framework + throw new ArgumentOutOfRangeException(nameof(dataOffset), dataOffset, message: null); + } + } + + return reader.Read(buffer, bufferOffset, length); } public virtual Stream GetStream(int ordinal) @@ -229,59 +237,65 @@ public virtual Stream GetStream(int ordinal) var blobDatabaseName = sqlite3_column_database_name(Handle, ordinal).utf8_to_string(); var blobTableName = sqlite3_column_table_name(Handle, ordinal).utf8_to_string(); - var rowidOrdinal = -1; - for (var i = 0; i < FieldCount; i++) + if (!_rowidOrdinal.HasValue) { - if (i == ordinal) - { - continue; - } + _rowidOrdinal = -1; - var databaseName = sqlite3_column_database_name(Handle, i).utf8_to_string(); - if (databaseName != blobDatabaseName) + for (var i = 0; i < FieldCount; i++) { - continue; + if (i == ordinal) + { + continue; + } + + var databaseName = sqlite3_column_database_name(Handle, i).utf8_to_string(); + if (databaseName != blobDatabaseName) + { + continue; + } + + var tableName = sqlite3_column_table_name(Handle, i).utf8_to_string(); + if (tableName != blobTableName) + { + continue; + } + + var columnName = sqlite3_column_origin_name(Handle, i).utf8_to_string(); + if (columnName == "rowid") + { + _rowidOrdinal = i; + break; + } + + var rc = sqlite3_table_column_metadata( + _connection.Handle, + databaseName, + tableName, + columnName, + out var dataType, + out var collSeq, + out var notNull, + out var primaryKey, + out var autoInc); + SqliteException.ThrowExceptionForRC(rc, _connection.Handle); + if (string.Equals(dataType, "INTEGER", StringComparison.OrdinalIgnoreCase) + && primaryKey != 0) + { + _rowidOrdinal = i; + break; + } } - var tableName = sqlite3_column_table_name(Handle, i).utf8_to_string(); - if (tableName != blobTableName) - { - continue; - } - - var columnName = sqlite3_column_origin_name(Handle, i).utf8_to_string(); - if (columnName == "rowid") - { - rowidOrdinal = i; - break; - } - - var rc = sqlite3_table_column_metadata( - _connection.Handle, - databaseName, - tableName, - columnName, - out var dataType, - out var collSeq, - out var notNull, - out var primaryKey, - out var autoInc); - SqliteException.ThrowExceptionForRC(rc, _connection.Handle); - if (string.Equals(dataType, "INTEGER", StringComparison.OrdinalIgnoreCase) - && primaryKey != 0) - { - rowidOrdinal = i; - break; - } + Debug.Assert(_rowidOrdinal.HasValue); } - if (rowidOrdinal < 0) + if (_rowidOrdinal.Value < 0) { return new MemoryStream(GetCachedBlob(ordinal), false); } var blobColumnName = sqlite3_column_origin_name(Handle, ordinal).utf8_to_string(); - var rowid = GetInt32(rowidOrdinal); + var rowid = GetInt32(_rowidOrdinal.Value); return new SqliteBlob(_connection, blobTableName, blobColumnName, rowid, readOnly: true); } diff --git a/test/Microsoft.Data.Sqlite.Tests/SqliteDataReaderTest.cs b/test/Microsoft.Data.Sqlite.Tests/SqliteDataReaderTest.cs index aca57006ddb..a6289e30f40 100644 --- a/test/Microsoft.Data.Sqlite.Tests/SqliteDataReaderTest.cs +++ b/test/Microsoft.Data.Sqlite.Tests/SqliteDataReaderTest.cs @@ -127,6 +127,27 @@ public void GetBytes_works() } } + [Fact] + public void GetBytes_works_streaming() + { + using (var connection = new SqliteConnection("Data Source=:memory:")) + { + connection.Open(); + + connection.ExecuteNonQuery("CREATE TABLE Data (Value); INSERT INTO Data VALUES (x'01020304');"); + + using (var reader = connection.ExecuteReader("SELECT rowid, Value FROM Data;")) + { + var hasData = reader.Read(); + Assert.True(hasData); + + var buffer = new byte[2]; + reader.GetBytes(1, 1, buffer, 0, buffer.Length); + Assert.Equal(new byte[] { 0x02, 0x03 }, buffer); + } + } + } + [Fact] public void GetBytes_NullBuffer() { @@ -252,6 +273,26 @@ public void GetChars_works_with_overflow() } } + [Fact] + public void GetChars_throws_when_dataOffset_out_of_range() + { + using (var connection = new SqliteConnection("Data Source=:memory:")) + { + connection.Open(); + + using (var reader = connection.ExecuteReader("SELECT 'test';")) + { + var hasData = reader.Read(); + Assert.True(hasData); + + var buffer = new char[1]; + var ex = Assert.Throws( + () => reader.GetChars(0, 5, buffer, 0, buffer.Length)); + Assert.Equal("dataOffset", ex.ParamName); + } + } + } + [Fact] public void GetChars_throws_when_closed() { @@ -262,6 +303,27 @@ public void GetChars_throws_when_closed() public void GetChars_throws_when_non_query() => X_throws_when_non_query(r => r.GetChars(0, 0, null, 0, 0)); + [Fact] + public void GetChars_works_streaming() + { + using (var connection = new SqliteConnection("Data Source=:memory:")) + { + connection.Open(); + + connection.ExecuteNonQuery("CREATE TABLE Data (Value); INSERT INTO Data VALUES ('test');"); + + using (var reader = connection.ExecuteReader("SELECT rowid, Value FROM Data;")) + { + var hasData = reader.Read(); + Assert.True(hasData); + + var buffer = new char[2]; + reader.GetChars(1, 1, buffer, 0, buffer.Length); + Assert.Equal(new[] { 'e', 's' }, buffer); + } + } + } + [Fact] public void GetStream_works() { @@ -388,6 +450,71 @@ public void GetStream_throws_when_closed() public void GetStream_throws_when_non_query() => X_throws_when_non_query(r => r.GetStream(0)); + [Fact] + public void GetTextReader_works() + { + using (var connection = new SqliteConnection("Data Source=:memory:")) + { + connection.Open(); + + using (var reader = connection.ExecuteReader("SELECT 'test';")) + { + var hasData = reader.Read(); + Assert.True(hasData); + + using (var textReader = reader.GetTextReader(0)) + { + Assert.IsType(Assert.IsType(textReader).BaseStream); + Assert.Equal("test", textReader.ReadToEnd()); + } + } + } + } + + [Fact] + public void GetTextReader_works_when_null() + { + using (var connection = new SqliteConnection("Data Source=:memory:")) + { + connection.Open(); + + using (var reader = connection.ExecuteReader("SELECT NULL;")) + { + var hasData = reader.Read(); + Assert.True(hasData); + + using (var textReader = reader.GetTextReader(0)) + { + Assert.IsType(textReader); + Assert.Empty(textReader.ReadToEnd()); + } + } + } + } + + [Fact] + public void GetTextReader_works_streaming() + { + using (var connection = new SqliteConnection("Data Source=:memory:")) + { + connection.Open(); + + connection.ExecuteNonQuery("CREATE TABLE Data (Value); INSERT INTO Data VALUES ('test');"); + + using (var reader = connection.ExecuteReader("SELECT rowid, Value FROM Data;")) + { + var hasData = reader.Read(); + Assert.True(hasData); + + using (var textReader = reader.GetTextReader(1)) + { + Assert.IsType(Assert.IsType(textReader).BaseStream); + Assert.Equal("test", textReader.ReadToEnd()); + } + } + } + } + [Fact] public void GetDateTime_works_with_text() => GetX_works(