From 95e629b7a1b31ce86465c76e69fe151fb7ea87b8 Mon Sep 17 00:00:00 2001 From: SpaceCheetah Date: Sat, 23 Mar 2024 01:05:31 -0700 Subject: [PATCH 1/7] Fix LiteFileStream.SetReadStreamPosition --- LiteDB/Client/Storage/LiteFileStream.Read.cs | 31 +++++++++++++++----- LiteDB/Client/Storage/LiteFileStream.cs | 2 ++ 2 files changed, 26 insertions(+), 7 deletions(-) diff --git a/LiteDB/Client/Storage/LiteFileStream.Read.cs b/LiteDB/Client/Storage/LiteFileStream.Read.cs index ed4d4e158..a9b1fd5cb 100644 --- a/LiteDB/Client/Storage/LiteFileStream.Read.cs +++ b/LiteDB/Client/Storage/LiteFileStream.Read.cs @@ -42,23 +42,40 @@ private byte[] GetChunkData(int index) .FindOne("_id = { f: @0, n: @1 }", _fileId, index); // if chunk is null there is no more chunks - return chunk?["data"].AsBinary; + byte[] result = chunk?["data"].AsBinary; + if (result != null) { + _chunkLengths[index] = result.Length; + } + return result; } private void SetReadStreamPosition(long newPosition) { - if (newPosition < 0 && newPosition > Length) + if (newPosition < 0 || newPosition > Length) { throw new ArgumentOutOfRangeException(); } _streamPosition = newPosition; // calculate new chunk position - _currentChunkIndex = (int)_streamPosition / MAX_CHUNK_SIZE; - _positionInChunk = (int)_streamPosition % MAX_CHUNK_SIZE; - - // get current chunk - _currentChunkData = this.GetChunkData(_currentChunkIndex); + long seekStreamPosition = 0; + int loadedChunk = _currentChunkIndex; + int newChunkIndex = 0; + while (seekStreamPosition <= _streamPosition) { + if (!_chunkLengths.ContainsKey(newChunkIndex)) { + loadedChunk = newChunkIndex; + _currentChunkData = GetChunkData(newChunkIndex); + } + seekStreamPosition += _chunkLengths[newChunkIndex]; + newChunkIndex++; + } + newChunkIndex--; + seekStreamPosition -= _chunkLengths[newChunkIndex]; + _positionInChunk = (int)(_streamPosition - seekStreamPosition); + _currentChunkIndex = newChunkIndex; + if (loadedChunk != _currentChunkIndex) { + _currentChunkData = GetChunkData(_currentChunkIndex); + } } } } \ No newline at end of file diff --git a/LiteDB/Client/Storage/LiteFileStream.cs b/LiteDB/Client/Storage/LiteFileStream.cs index 1b476f645..3b5bbfc82 100644 --- a/LiteDB/Client/Storage/LiteFileStream.cs +++ b/LiteDB/Client/Storage/LiteFileStream.cs @@ -1,4 +1,5 @@ using System; +using System.Collections.Generic; using System.IO; using System.Linq; using static LiteDB.Constants; @@ -23,6 +24,7 @@ public partial class LiteFileStream : Stream private byte[] _currentChunkData = null; private int _positionInChunk = 0; private MemoryStream _buffer; + private Dictionary _chunkLengths = new Dictionary(); internal LiteFileStream(ILiteCollection> files, ILiteCollection chunks, LiteFileInfo file, BsonValue fileId, FileAccess mode) { From 20a88f3159ecf2036434a3e339d1c69a29cb9301 Mon Sep 17 00:00:00 2001 From: SpaceCheetah Date: Tue, 4 Jun 2024 13:33:41 -0700 Subject: [PATCH 2/7] Use TryGetValue --- LiteDB/Client/Storage/LiteFileStream.Read.cs | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/LiteDB/Client/Storage/LiteFileStream.Read.cs b/LiteDB/Client/Storage/LiteFileStream.Read.cs index a9b1fd5cb..4ba0208f7 100644 --- a/LiteDB/Client/Storage/LiteFileStream.Read.cs +++ b/LiteDB/Client/Storage/LiteFileStream.Read.cs @@ -62,11 +62,14 @@ private void SetReadStreamPosition(long newPosition) int loadedChunk = _currentChunkIndex; int newChunkIndex = 0; while (seekStreamPosition <= _streamPosition) { - if (!_chunkLengths.ContainsKey(newChunkIndex)) { + if (_chunkLengths.TryGetValue(newChunkIndex, out long length)) { + seekStreamPosition += length; + } + else { loadedChunk = newChunkIndex; _currentChunkData = GetChunkData(newChunkIndex); + seekStreamPosition += _currentChunkData.Length; } - seekStreamPosition += _chunkLengths[newChunkIndex]; newChunkIndex++; } newChunkIndex--; From 0ded5deea41f1dd0a64639e899abb18f3b481705 Mon Sep 17 00:00:00 2001 From: SpaceCheetah Date: Tue, 4 Jun 2024 14:16:45 -0700 Subject: [PATCH 3/7] Add tests --- LiteDB.Tests/Issues/Issue_2458_Tests.cs | 54 +++++++++++++++++++++++++ 1 file changed, 54 insertions(+) create mode 100644 LiteDB.Tests/Issues/Issue_2458_Tests.cs diff --git a/LiteDB.Tests/Issues/Issue_2458_Tests.cs b/LiteDB.Tests/Issues/Issue_2458_Tests.cs new file mode 100644 index 000000000..a05d7df46 --- /dev/null +++ b/LiteDB.Tests/Issues/Issue_2458_Tests.cs @@ -0,0 +1,54 @@ +using System; +using System.IO; +using Xunit; + +namespace LiteDB.Tests.Issues; + +public class Issue_2458_Tests +{ + [Fact] + public void NegativeSeekFails() + { + using var db = new LiteDatabase(":memory:"); + var fs = db.FileStorage; + AddTestFile("test", 1, fs); + using Stream stream = fs.OpenRead("test"); + Assert.Throws(() => stream.Position = -1); + } + + //https://learn.microsoft.com/en-us/dotnet/api/system.io.stream.position?view=net-8.0 says seeking to a position + //beyond the end of a stream is supported, so implementations should support it (error on read). + [Fact] + public void SeekPastFileSucceds() + { + using var db = new LiteDatabase(":memory:"); + var fs = db.FileStorage; + AddTestFile("test", 1, fs); + using Stream stream = fs.OpenRead("test"); + stream.Position = Int32.MaxValue; + } + + [Fact] + public void SeekShortChunks() + { + using var db = new LiteDatabase(":memory:"); + var fs = db.FileStorage; + using(Stream writeStream = fs.OpenWrite("test", "test")) + { + writeStream.WriteByte(0); + writeStream.Flush(); //Create single-byte chunk just containing a 0 + writeStream.WriteByte(1); + writeStream.Flush(); + writeStream.WriteByte(2); + } + using Stream readStream = fs.OpenRead("test"); + readStream.Position = 2; + Assert.Equal(2, readStream.ReadByte()); + } + + private void AddTestFile(string id, long length, ILiteStorage fs) + { + using Stream writeStream = fs.OpenWrite(id, id); + writeStream.Write(new byte[length]); + } +} \ No newline at end of file From 0b3a76a0981dfc73c5d787f4e28c52c51d5b9ef0 Mon Sep 17 00:00:00 2001 From: SpaceCheetah Date: Tue, 4 Jun 2024 14:21:23 -0700 Subject: [PATCH 4/7] rename to match convention --- LiteDB.Tests/Issues/{Issue_2458_Tests.cs => Issue2458_Tests.cs} | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename LiteDB.Tests/Issues/{Issue_2458_Tests.cs => Issue2458_Tests.cs} (98%) diff --git a/LiteDB.Tests/Issues/Issue_2458_Tests.cs b/LiteDB.Tests/Issues/Issue2458_Tests.cs similarity index 98% rename from LiteDB.Tests/Issues/Issue_2458_Tests.cs rename to LiteDB.Tests/Issues/Issue2458_Tests.cs index a05d7df46..113132f6c 100644 --- a/LiteDB.Tests/Issues/Issue_2458_Tests.cs +++ b/LiteDB.Tests/Issues/Issue2458_Tests.cs @@ -4,7 +4,7 @@ namespace LiteDB.Tests.Issues; -public class Issue_2458_Tests +public class Issue2458_Tests { [Fact] public void NegativeSeekFails() From 9473c1a04718a4ac06c90eadaa2cf93624255cf1 Mon Sep 17 00:00:00 2001 From: SpaceCheetah Date: Tue, 4 Jun 2024 14:32:26 -0700 Subject: [PATCH 5/7] Support seeking past the end of a file --- LiteDB/Client/Storage/LiteFileStream.Read.cs | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/LiteDB/Client/Storage/LiteFileStream.Read.cs b/LiteDB/Client/Storage/LiteFileStream.Read.cs index 4ba0208f7..8e4d882d2 100644 --- a/LiteDB/Client/Storage/LiteFileStream.Read.cs +++ b/LiteDB/Client/Storage/LiteFileStream.Read.cs @@ -10,6 +10,9 @@ public partial class LiteFileStream : Stream public override int Read(byte[] buffer, int offset, int count) { if (_mode != FileAccess.Read) throw new NotSupportedException(); + if (_streamPosition == Length) { + return 0; + } var bytesLeft = count; @@ -51,10 +54,14 @@ private byte[] GetChunkData(int index) private void SetReadStreamPosition(long newPosition) { - if (newPosition < 0 || newPosition > Length) + if (newPosition < 0) { throw new ArgumentOutOfRangeException(); } + if (newPosition >= Length) { + _streamPosition = Length; + return; + } _streamPosition = newPosition; // calculate new chunk position From e1d2bce0628696f24c0d10dcb42d0a5208daf057 Mon Sep 17 00:00:00 2001 From: SpaceCheetah Date: Mon, 10 Jun 2024 23:53:01 -0700 Subject: [PATCH 6/7] Correct brace format --- LiteDB/Client/Storage/LiteFileStream.Read.cs | 21 +++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/LiteDB/Client/Storage/LiteFileStream.Read.cs b/LiteDB/Client/Storage/LiteFileStream.Read.cs index 8e4d882d2..9c77b8125 100644 --- a/LiteDB/Client/Storage/LiteFileStream.Read.cs +++ b/LiteDB/Client/Storage/LiteFileStream.Read.cs @@ -10,7 +10,8 @@ public partial class LiteFileStream : Stream public override int Read(byte[] buffer, int offset, int count) { if (_mode != FileAccess.Read) throw new NotSupportedException(); - if (_streamPosition == Length) { + if (_streamPosition == Length) + { return 0; } @@ -46,7 +47,8 @@ private byte[] GetChunkData(int index) // if chunk is null there is no more chunks byte[] result = chunk?["data"].AsBinary; - if (result != null) { + if (result != null) + { _chunkLengths[index] = result.Length; } return result; @@ -58,7 +60,8 @@ private void SetReadStreamPosition(long newPosition) { throw new ArgumentOutOfRangeException(); } - if (newPosition >= Length) { + if (newPosition >= Length) + { _streamPosition = Length; return; } @@ -68,11 +71,14 @@ private void SetReadStreamPosition(long newPosition) long seekStreamPosition = 0; int loadedChunk = _currentChunkIndex; int newChunkIndex = 0; - while (seekStreamPosition <= _streamPosition) { - if (_chunkLengths.TryGetValue(newChunkIndex, out long length)) { + while (seekStreamPosition <= _streamPosition) + { + if (_chunkLengths.TryGetValue(newChunkIndex, out long length)) + { seekStreamPosition += length; } - else { + else + { loadedChunk = newChunkIndex; _currentChunkData = GetChunkData(newChunkIndex); seekStreamPosition += _currentChunkData.Length; @@ -83,7 +89,8 @@ private void SetReadStreamPosition(long newPosition) seekStreamPosition -= _chunkLengths[newChunkIndex]; _positionInChunk = (int)(_streamPosition - seekStreamPosition); _currentChunkIndex = newChunkIndex; - if (loadedChunk != _currentChunkIndex) { + if (loadedChunk != _currentChunkIndex) + { _currentChunkData = GetChunkData(_currentChunkIndex); } } From 4d58c6900dfc5e010e74a9f3903dff3b3d142731 Mon Sep 17 00:00:00 2001 From: SpaceCheetah Date: Tue, 11 Jun 2024 14:50:56 -0700 Subject: [PATCH 7/7] More style changes --- LiteDB/Client/Storage/LiteFileStream.Read.cs | 3 +++ LiteDB/Client/Storage/LiteFileStream.cs | 2 -- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/LiteDB/Client/Storage/LiteFileStream.Read.cs b/LiteDB/Client/Storage/LiteFileStream.Read.cs index 9c77b8125..0b6236736 100644 --- a/LiteDB/Client/Storage/LiteFileStream.Read.cs +++ b/LiteDB/Client/Storage/LiteFileStream.Read.cs @@ -1,4 +1,5 @@ using System; +using System.Collections.Generic; using System.IO; using System.Linq; using static LiteDB.Constants; @@ -7,6 +8,7 @@ namespace LiteDB { public partial class LiteFileStream : Stream { + private Dictionary _chunkLengths = new Dictionary(); public override int Read(byte[] buffer, int offset, int count) { if (_mode != FileAccess.Read) throw new NotSupportedException(); @@ -85,6 +87,7 @@ private void SetReadStreamPosition(long newPosition) } newChunkIndex++; } + newChunkIndex--; seekStreamPosition -= _chunkLengths[newChunkIndex]; _positionInChunk = (int)(_streamPosition - seekStreamPosition); diff --git a/LiteDB/Client/Storage/LiteFileStream.cs b/LiteDB/Client/Storage/LiteFileStream.cs index 3b5bbfc82..1b476f645 100644 --- a/LiteDB/Client/Storage/LiteFileStream.cs +++ b/LiteDB/Client/Storage/LiteFileStream.cs @@ -1,5 +1,4 @@ using System; -using System.Collections.Generic; using System.IO; using System.Linq; using static LiteDB.Constants; @@ -24,7 +23,6 @@ public partial class LiteFileStream : Stream private byte[] _currentChunkData = null; private int _positionInChunk = 0; private MemoryStream _buffer; - private Dictionary _chunkLengths = new Dictionary(); internal LiteFileStream(ILiteCollection> files, ILiteCollection chunks, LiteFileInfo file, BsonValue fileId, FileAccess mode) {