Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Microsoft.Data.Sqlite: Bring back the SqliteBlob optimizations #20092

Merged
merged 1 commit into from
Feb 28, 2020
Merged

Microsoft.Data.Sqlite: Bring back the SqliteBlob optimizations #20092

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 10 additions & 0 deletions src/Microsoft.Data.Sqlite.Core/SqliteDataReader.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -544,6 +544,16 @@ public override Stream GetStream(int ordinal)
? throw new InvalidOperationException(Resources.NoData)
: _record.GetStream(ordinal);

/// <summary>
/// Retrieves data as a <see cref="TextReader" />.
/// </summary>
/// <param name="ordinal">The zero-based column ordinal.</param>
/// <returns>The returned object.</returns>
public override TextReader GetTextReader(int ordinal)
=> IsDBNull(ordinal)
? (TextReader)new StringReader(string.Empty)
: new StreamReader(GetStream(ordinal), Encoding.UTF8);

/// <summary>
/// Gets the value of the specified column.
/// </summary>
Expand Down
118 changes: 66 additions & 52 deletions src/Microsoft.Data.Sqlite.Core/SqliteDataRecord.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
using System.Diagnostics;
using System.IO;
using System.Linq;
using System.Text;
using Microsoft.Data.Sqlite.Properties;
using SQLitePCL;
using static SQLitePCL.raw;
Expand All @@ -17,6 +18,7 @@ internal class SqliteDataRecord : SqliteValueReader, IDisposable
private readonly byte[][] _blobCache;
private readonly int?[] _typeCache;
private bool _stepped;
private int? _rowidOrdinal;

public SqliteDataRecord(sqlite3_stmt stmt, bool hasRows, SqliteConnection connection)
{
Expand Down Expand Up @@ -196,26 +198,32 @@ public static Type GetFieldType(string type)

public virtual long GetBytes(int ordinal, long dataOffset, byte[] buffer, int bufferOffset, int length)
{
var blob = GetCachedBlob(ordinal);
using var stream = GetStream(ordinal);

long bytesToRead = blob.Length - dataOffset;
if (buffer != null)
if (buffer == null)
{
bytesToRead = Math.Min(bytesToRead, length);
Array.Copy(blob, dataOffset, buffer, bufferOffset, bytesToRead);
return stream.Length - dataOffset;
}

return bytesToRead;
stream.Position = dataOffset;

return stream.Read(buffer, bufferOffset, length);
}

public virtual long GetChars(int ordinal, long dataOffset, char[] buffer, int bufferOffset, int length)
{
var text = GetString(ordinal);
using var reader = new StreamReader(GetStream(ordinal), Encoding.UTF8);

int charsToRead = text.Length - (int)dataOffset;
charsToRead = Math.Min(charsToRead, length);
text.CopyTo((int)dataOffset, buffer, bufferOffset, charsToRead);
return charsToRead;
for (var position = 0; position < dataOffset; position++)
{
if (reader.Read() == -1)
{
// NB: Message is provided by the framework
throw new ArgumentOutOfRangeException(nameof(dataOffset), dataOffset, message: null);
}
}

return reader.Read(buffer, bufferOffset, length);
}

public virtual Stream GetStream(int ordinal)
Expand All @@ -229,59 +237,65 @@ public virtual Stream GetStream(int ordinal)
var blobDatabaseName = sqlite3_column_database_name(Handle, ordinal).utf8_to_string();
var blobTableName = sqlite3_column_table_name(Handle, ordinal).utf8_to_string();

var rowidOrdinal = -1;
for (var i = 0; i < FieldCount; i++)
if (!_rowidOrdinal.HasValue)
{
if (i == ordinal)
{
continue;
}
_rowidOrdinal = -1;

var databaseName = sqlite3_column_database_name(Handle, i).utf8_to_string();
if (databaseName != blobDatabaseName)
for (var i = 0; i < FieldCount; i++)
{
continue;
if (i == ordinal)
{
continue;
}

var databaseName = sqlite3_column_database_name(Handle, i).utf8_to_string();
if (databaseName != blobDatabaseName)
{
continue;
}

var tableName = sqlite3_column_table_name(Handle, i).utf8_to_string();
if (tableName != blobTableName)
{
continue;
}

var columnName = sqlite3_column_origin_name(Handle, i).utf8_to_string();
if (columnName == "rowid")
{
_rowidOrdinal = i;
break;
}

var rc = sqlite3_table_column_metadata(
_connection.Handle,
databaseName,
tableName,
columnName,
out var dataType,
out var collSeq,
out var notNull,
out var primaryKey,
out var autoInc);
SqliteException.ThrowExceptionForRC(rc, _connection.Handle);
if (string.Equals(dataType, "INTEGER", StringComparison.OrdinalIgnoreCase)
&& primaryKey != 0)
{
_rowidOrdinal = i;
break;
}
}

var tableName = sqlite3_column_table_name(Handle, i).utf8_to_string();
if (tableName != blobTableName)
{
continue;
}

var columnName = sqlite3_column_origin_name(Handle, i).utf8_to_string();
if (columnName == "rowid")
{
rowidOrdinal = i;
break;
}

var rc = sqlite3_table_column_metadata(
_connection.Handle,
databaseName,
tableName,
columnName,
out var dataType,
out var collSeq,
out var notNull,
out var primaryKey,
out var autoInc);
SqliteException.ThrowExceptionForRC(rc, _connection.Handle);
if (string.Equals(dataType, "INTEGER", StringComparison.OrdinalIgnoreCase)
&& primaryKey != 0)
{
rowidOrdinal = i;
break;
}
Debug.Assert(_rowidOrdinal.HasValue);
}

if (rowidOrdinal < 0)
if (_rowidOrdinal.Value < 0)
{
return new MemoryStream(GetCachedBlob(ordinal), false);
}

var blobColumnName = sqlite3_column_origin_name(Handle, ordinal).utf8_to_string();
var rowid = GetInt32(rowidOrdinal);
var rowid = GetInt32(_rowidOrdinal.Value);

return new SqliteBlob(_connection, blobTableName, blobColumnName, rowid, readOnly: true);
}
Expand Down
127 changes: 127 additions & 0 deletions test/Microsoft.Data.Sqlite.Tests/SqliteDataReaderTest.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -127,6 +127,27 @@ public void GetBytes_works()
}
}

[Fact]
public void GetBytes_works_streaming()
{
using (var connection = new SqliteConnection("Data Source=:memory:"))
{
connection.Open();

connection.ExecuteNonQuery("CREATE TABLE Data (Value); INSERT INTO Data VALUES (x'01020304');");

using (var reader = connection.ExecuteReader("SELECT rowid, Value FROM Data;"))
{
var hasData = reader.Read();
Assert.True(hasData);

var buffer = new byte[2];
reader.GetBytes(1, 1, buffer, 0, buffer.Length);
Assert.Equal(new byte[] { 0x02, 0x03 }, buffer);
}
}
}

[Fact]
public void GetBytes_NullBuffer()
{
Expand Down Expand Up @@ -252,6 +273,26 @@ public void GetChars_works_with_overflow()
}
}

[Fact]
public void GetChars_throws_when_dataOffset_out_of_range()
{
using (var connection = new SqliteConnection("Data Source=:memory:"))
{
connection.Open();

using (var reader = connection.ExecuteReader("SELECT 'test';"))
{
var hasData = reader.Read();
Assert.True(hasData);

var buffer = new char[1];
var ex = Assert.Throws<ArgumentOutOfRangeException>(
() => reader.GetChars(0, 5, buffer, 0, buffer.Length));
Assert.Equal("dataOffset", ex.ParamName);
}
}
}

[Fact]
public void GetChars_throws_when_closed()
{
Expand All @@ -262,6 +303,27 @@ public void GetChars_throws_when_closed()
public void GetChars_throws_when_non_query()
=> X_throws_when_non_query(r => r.GetChars(0, 0, null, 0, 0));

[Fact]
public void GetChars_works_streaming()
{
using (var connection = new SqliteConnection("Data Source=:memory:"))
{
connection.Open();

connection.ExecuteNonQuery("CREATE TABLE Data (Value); INSERT INTO Data VALUES ('test');");

using (var reader = connection.ExecuteReader("SELECT rowid, Value FROM Data;"))
{
var hasData = reader.Read();
Assert.True(hasData);

var buffer = new char[2];
reader.GetChars(1, 1, buffer, 0, buffer.Length);
Assert.Equal(new[] { 'e', 's' }, buffer);
}
}
}

[Fact]
public void GetStream_works()
{
Expand Down Expand Up @@ -388,6 +450,71 @@ public void GetStream_throws_when_closed()
public void GetStream_throws_when_non_query()
=> X_throws_when_non_query(r => r.GetStream(0));

[Fact]
public void GetTextReader_works()
{
using (var connection = new SqliteConnection("Data Source=:memory:"))
{
connection.Open();

using (var reader = connection.ExecuteReader("SELECT 'test';"))
{
var hasData = reader.Read();
Assert.True(hasData);

using (var textReader = reader.GetTextReader(0))
{
Assert.IsType<MemoryStream>(Assert.IsType<StreamReader>(textReader).BaseStream);
Assert.Equal("test", textReader.ReadToEnd());
}
}
}
}

[Fact]
public void GetTextReader_works_when_null()
{
using (var connection = new SqliteConnection("Data Source=:memory:"))
{
connection.Open();

using (var reader = connection.ExecuteReader("SELECT NULL;"))
{
var hasData = reader.Read();
Assert.True(hasData);

using (var textReader = reader.GetTextReader(0))
{
Assert.IsType<StringReader>(textReader);
Assert.Empty(textReader.ReadToEnd());
}
}
}
}

[Fact]
public void GetTextReader_works_streaming()
{
using (var connection = new SqliteConnection("Data Source=:memory:"))
{
connection.Open();

connection.ExecuteNonQuery("CREATE TABLE Data (Value); INSERT INTO Data VALUES ('test');");

using (var reader = connection.ExecuteReader("SELECT rowid, Value FROM Data;"))
{
var hasData = reader.Read();
Assert.True(hasData);

using (var textReader = reader.GetTextReader(1))
{
Assert.IsType<SqliteBlob>(Assert.IsType<StreamReader>(textReader).BaseStream);
Assert.Equal("test", textReader.ReadToEnd());
}
}
}
}

[Fact]
public void GetDateTime_works_with_text()
=> GetX_works(
Expand Down