From 771acc949b298366ed1cda553a107b2edb0af0ae Mon Sep 17 00:00:00 2001 From: Jimmy Date: Fri, 5 Jul 2024 11:34:23 +0800 Subject: [PATCH 1/6] test --- tests/Neo.Plugins.Storage.Tests/StoreTest.cs | 48 ++++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/tests/Neo.Plugins.Storage.Tests/StoreTest.cs b/tests/Neo.Plugins.Storage.Tests/StoreTest.cs index 2773f92e80..02992b612f 100644 --- a/tests/Neo.Plugins.Storage.Tests/StoreTest.cs +++ b/tests/Neo.Plugins.Storage.Tests/StoreTest.cs @@ -63,6 +63,54 @@ public void TestLevelDb() TestPersistenceRead(plugin.GetStore(path_leveldb), false); } + [TestMethod] + public void TestLevelDbSnapshot() + { + using var plugin = new LevelDBStore(); + using var store = plugin.GetStore(path_leveldb); + + var snapshot = store.GetSnapshot(); + + var testKey = new byte[] { 0x01, 0x02, 0x03 }; + var testValue = new byte[] { 0x04, 0x05, 0x06 }; + + snapshot.Put(testKey,testValue); + // Data saved to the leveldb snapshot shall not be visible to the store + Assert.IsNull(snapshot.TryGet(testKey)); + snapshot.Commit(); + + // After commit, the data shall be visible to the store and to the snapshot + CollectionAssert.AreEqual(testValue, snapshot.TryGet(testKey)); + var b = store.TryGet(testKey); + CollectionAssert.AreEqual(testValue,b ); + + snapshot.Dispose(); + } + + [TestMethod] + public void TestLevelDbMultiSnapshot() + { + using var plugin = new LevelDBStore(); + using var store = plugin.GetStore(path_leveldb); + + var snapshot = store.GetSnapshot(); + var snapshot2 = store.GetSnapshot(); + + var testKey = new byte[] { 0x01, 0x02, 0x03 }; + var testValue = new byte[] { 0x04, 0x05, 0x06 }; + + snapshot.Put(testKey,testValue); + CollectionAssert.AreEqual(testValue, snapshot.TryGet(testKey)); + snapshot.Commit(); + CollectionAssert.AreEqual(testValue, store.TryGet(testKey)); + + var ret = snapshot2.TryGet(testKey); + Assert.IsNull(ret); + + snapshot.Dispose(); + snapshot2.Dispose(); + } + [TestMethod] public void TestRocksDb() { From 2e6979c5039130a18e4713f2138397444304a2ff Mon Sep 17 00:00:00 2001 From: Jimmy Date: Fri, 5 Jul 2024 12:12:51 +0800 Subject: [PATCH 2/6] fix snapshot issue and add tests --- .../LevelDBStore/Plugins/Storage/Snapshot.cs | 7 +- .../RocksDBStore/Plugins/Storage/Snapshot.cs | 4 +- tests/Neo.Plugins.Storage.Tests/StoreTest.cs | 127 ++++++++++++++---- 3 files changed, 106 insertions(+), 32 deletions(-) diff --git a/src/Plugins/LevelDBStore/Plugins/Storage/Snapshot.cs b/src/Plugins/LevelDBStore/Plugins/Storage/Snapshot.cs index 0b0a63b885..9606afd764 100644 --- a/src/Plugins/LevelDBStore/Plugins/Storage/Snapshot.cs +++ b/src/Plugins/LevelDBStore/Plugins/Storage/Snapshot.cs @@ -48,7 +48,8 @@ public void Dispose() public IEnumerable<(byte[] Key, byte[] Value)> Seek(byte[] prefix, SeekDirection direction = SeekDirection.Forward) { - return db.Seek(options, prefix, direction, (k, v) => (k, v)); + return db.Seek(options, prefix, direction, (k, v) => (k, v)) + ?? db.Seek(ReadOptions.Default, prefix, direction, (k, v) => (k, v)); } public void Put(byte[] key, byte[] value) @@ -58,12 +59,12 @@ public void Put(byte[] key, byte[] value) public bool Contains(byte[] key) { - return db.Contains(options, key); + return db.Contains(options, key) || db.Contains(ReadOptions.Default, key); } public byte[] TryGet(byte[] key) { - return db.Get(options, key); + return db.Get(options, key) ?? db.Get(ReadOptions.Default, key); } } } diff --git a/src/Plugins/RocksDBStore/Plugins/Storage/Snapshot.cs b/src/Plugins/RocksDBStore/Plugins/Storage/Snapshot.cs index 7423f6ae4a..1078ea7778 100644 --- a/src/Plugins/RocksDBStore/Plugins/Storage/Snapshot.cs +++ b/src/Plugins/RocksDBStore/Plugins/Storage/Snapshot.cs @@ -65,12 +65,12 @@ public IEnumerable<(byte[] Key, byte[] Value)> Seek(byte[] keyOrPrefix, SeekDire public bool Contains(byte[] key) { - return db.Get(key, Array.Empty(), 0, 0, readOptions: options) >= 0; + return db.Get(key, Array.Empty(), 0, 0, readOptions: options) >= 0 || db.Get(key, Array.Empty(), 0, 0) >= 0; } public byte[] TryGet(byte[] key) { - return db.Get(key, readOptions: options); + return db.Get(key, readOptions: options) ?? db.Get(key); } public void Dispose() diff --git a/tests/Neo.Plugins.Storage.Tests/StoreTest.cs b/tests/Neo.Plugins.Storage.Tests/StoreTest.cs index 02992b612f..405541fba5 100644 --- a/tests/Neo.Plugins.Storage.Tests/StoreTest.cs +++ b/tests/Neo.Plugins.Storage.Tests/StoreTest.cs @@ -21,12 +21,21 @@ public class StoreTest { private const string path_leveldb = "Data_LevelDB_UT"; private const string path_rocksdb = "Data_RocksDB_UT"; + private static LevelDBStore levelDbStore; + private static RocksDBStore rocksDBStore; [TestInitialize] public void OnStart() { - if (Directory.Exists(path_leveldb)) Directory.Delete(path_leveldb, true); - if (Directory.Exists(path_rocksdb)) Directory.Delete(path_rocksdb, true); + levelDbStore ??= new LevelDBStore(); + rocksDBStore ??= new RocksDBStore(); + ReSetStore(); + } + + [TestCleanup] + public void OnEnd() + { + ReSetStore(); } [TestMethod] @@ -49,40 +58,46 @@ public void TestMemory() [TestMethod] public void TestLevelDb() { - using var plugin = new LevelDBStore(); - TestPersistenceDelete(plugin.GetStore(path_leveldb)); + ReSetStore(); + TestPersistenceDelete(levelDbStore.GetStore(path_leveldb)); // Test all with the same store - TestStorage(plugin.GetStore(path_leveldb)); + TestStorage(levelDbStore.GetStore(path_leveldb)); // Test with different storages - TestPersistenceWrite(plugin.GetStore(path_leveldb)); - TestPersistenceRead(plugin.GetStore(path_leveldb), true); - TestPersistenceDelete(plugin.GetStore(path_leveldb)); - TestPersistenceRead(plugin.GetStore(path_leveldb), false); + TestPersistenceWrite(levelDbStore.GetStore(path_leveldb)); + TestPersistenceRead(levelDbStore.GetStore(path_leveldb), true); + TestPersistenceDelete(levelDbStore.GetStore(path_leveldb)); + TestPersistenceRead(levelDbStore.GetStore(path_leveldb), false); } [TestMethod] public void TestLevelDbSnapshot() { - using var plugin = new LevelDBStore(); - using var store = plugin.GetStore(path_leveldb); + ReSetStore(); + using var store = levelDbStore.GetStore(path_leveldb); var snapshot = store.GetSnapshot(); var testKey = new byte[] { 0x01, 0x02, 0x03 }; var testValue = new byte[] { 0x04, 0x05, 0x06 }; - snapshot.Put(testKey,testValue); + snapshot.Put(testKey, testValue); // Data saved to the leveldb snapshot shall not be visible to the store Assert.IsNull(snapshot.TryGet(testKey)); + + // Value is in the write batch, not visible to the store and snapshot + Assert.AreEqual(false, snapshot.Contains(testKey)); + Assert.AreEqual(false, store.Contains(testKey)); + snapshot.Commit(); // After commit, the data shall be visible to the store and to the snapshot CollectionAssert.AreEqual(testValue, snapshot.TryGet(testKey)); - var b = store.TryGet(testKey); - CollectionAssert.AreEqual(testValue,b ); + CollectionAssert.AreEqual(testValue, store.TryGet(testKey)); + Assert.AreEqual(true, snapshot.Contains(testKey)); + Assert.AreEqual(true, store.Contains(testKey)); snapshot.Dispose(); } @@ -90,8 +105,8 @@ public void TestLevelDbSnapshot() [TestMethod] public void TestLevelDbMultiSnapshot() { - using var plugin = new LevelDBStore(); - using var store = plugin.GetStore(path_leveldb); + ReSetStore(); + using var store = levelDbStore.GetStore(path_leveldb); var snapshot = store.GetSnapshot(); var snapshot2 = store.GetSnapshot(); @@ -99,13 +114,12 @@ public void TestLevelDbMultiSnapshot() var testKey = new byte[] { 0x01, 0x02, 0x03 }; var testValue = new byte[] { 0x04, 0x05, 0x06 }; - snapshot.Put(testKey,testValue); - CollectionAssert.AreEqual(testValue, snapshot.TryGet(testKey)); + snapshot.Put(testKey, testValue); snapshot.Commit(); CollectionAssert.AreEqual(testValue, store.TryGet(testKey)); - var ret = snapshot2.TryGet(testKey); - Assert.IsNull(ret); + // Data saved to the leveldb from snapshot1 shall also be visible to snapshot2 + CollectionAssert.AreEqual(testValue, snapshot2.TryGet(testKey)); snapshot.Dispose(); snapshot2.Dispose(); @@ -114,18 +128,71 @@ public void TestLevelDbMultiSnapshot() [TestMethod] public void TestRocksDb() { - using var plugin = new RocksDBStore(); - TestPersistenceDelete(plugin.GetStore(path_rocksdb)); + TestPersistenceDelete(rocksDBStore.GetStore(path_rocksdb)); // Test all with the same store - TestStorage(plugin.GetStore(path_rocksdb)); + TestStorage(rocksDBStore.GetStore(path_rocksdb)); // Test with different storages - TestPersistenceWrite(plugin.GetStore(path_rocksdb)); - TestPersistenceRead(plugin.GetStore(path_rocksdb), true); - TestPersistenceDelete(plugin.GetStore(path_rocksdb)); - TestPersistenceRead(plugin.GetStore(path_rocksdb), false); + TestPersistenceWrite(rocksDBStore.GetStore(path_rocksdb)); + TestPersistenceRead(rocksDBStore.GetStore(path_rocksdb), true); + TestPersistenceDelete(rocksDBStore.GetStore(path_rocksdb)); + TestPersistenceRead(rocksDBStore.GetStore(path_rocksdb), false); + } + + [TestMethod] + public void TestRocksDbSnapshot() + { + ReSetStore(); + using var store = rocksDBStore.GetStore(path_leveldb); + + var snapshot = store.GetSnapshot(); + + var testKey = new byte[] { 0x01, 0x02, 0x03 }; + var testValue = new byte[] { 0x04, 0x05, 0x06 }; + + snapshot.Put(testKey, testValue); + // Data saved to the leveldb snapshot shall not be visible to the store + Assert.IsNull(snapshot.TryGet(testKey)); + + // Value is in the write batch, not visible to the store and snapshot + Assert.AreEqual(false, snapshot.Contains(testKey)); + Assert.AreEqual(false, store.Contains(testKey)); + + snapshot.Commit(); + + // After commit, the data shall be visible to the store and to the snapshot + CollectionAssert.AreEqual(testValue, snapshot.TryGet(testKey)); + CollectionAssert.AreEqual(testValue, store.TryGet(testKey)); + Assert.AreEqual(true, snapshot.Contains(testKey)); + Assert.AreEqual(true, store.Contains(testKey)); + + snapshot.Dispose(); + } + + [TestMethod] + public void TestRocksDbMultiSnapshot() + { + ReSetStore(); + using var store = rocksDBStore.GetStore(path_leveldb); + + var snapshot = store.GetSnapshot(); + var snapshot2 = store.GetSnapshot(); + + var testKey = new byte[] { 0x01, 0x02, 0x03 }; + var testValue = new byte[] { 0x04, 0x05, 0x06 }; + + snapshot.Put(testKey, testValue); + snapshot.Commit(); + CollectionAssert.AreEqual(testValue, store.TryGet(testKey)); + + // Data saved to the leveldb from snapshot1 shall also be visible to snapshot2 + var ret = snapshot2.TryGet(testKey); + Assert.IsNull(ret); + + snapshot.Dispose(); + snapshot2.Dispose(); } /// @@ -244,5 +311,11 @@ private void TestPersistenceRead(IStore store, bool shouldExist) else Assert.IsNull(ret); } } + + private void ReSetStore() + { + if (Directory.Exists(path_leveldb)) Directory.Delete(path_leveldb, true); + if (Directory.Exists(path_rocksdb)) Directory.Delete(path_rocksdb, true); + } } } From 8f3755872ccd5bc8903af27a9094ce50133c9fe2 Mon Sep 17 00:00:00 2001 From: Jimmy Date: Fri, 5 Jul 2024 12:14:16 +0800 Subject: [PATCH 3/6] fix test --- tests/Neo.Plugins.Storage.Tests/StoreTest.cs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/Neo.Plugins.Storage.Tests/StoreTest.cs b/tests/Neo.Plugins.Storage.Tests/StoreTest.cs index 405541fba5..d75c3d75d2 100644 --- a/tests/Neo.Plugins.Storage.Tests/StoreTest.cs +++ b/tests/Neo.Plugins.Storage.Tests/StoreTest.cs @@ -188,8 +188,7 @@ public void TestRocksDbMultiSnapshot() CollectionAssert.AreEqual(testValue, store.TryGet(testKey)); // Data saved to the leveldb from snapshot1 shall also be visible to snapshot2 - var ret = snapshot2.TryGet(testKey); - Assert.IsNull(ret); + CollectionAssert.AreEqual(testValue, snapshot2.TryGet(testKey)); snapshot.Dispose(); snapshot2.Dispose(); From 20453f6f77421b724b7257a2fb6d1eccc1fcfd82 Mon Sep 17 00:00:00 2001 From: Jimmy Date: Sat, 6 Jul 2024 08:56:12 +0800 Subject: [PATCH 4/6] apply old snapshot --- .../LevelDBStore/Plugins/Storage/Snapshot.cs | 7 +++--- .../RocksDBStore/Plugins/Storage/Snapshot.cs | 4 +-- tests/Neo.Plugins.Storage.Tests/StoreTest.cs | 25 +++++++++++-------- 3 files changed, 19 insertions(+), 17 deletions(-) diff --git a/src/Plugins/LevelDBStore/Plugins/Storage/Snapshot.cs b/src/Plugins/LevelDBStore/Plugins/Storage/Snapshot.cs index 9606afd764..0b0a63b885 100644 --- a/src/Plugins/LevelDBStore/Plugins/Storage/Snapshot.cs +++ b/src/Plugins/LevelDBStore/Plugins/Storage/Snapshot.cs @@ -48,8 +48,7 @@ public void Dispose() public IEnumerable<(byte[] Key, byte[] Value)> Seek(byte[] prefix, SeekDirection direction = SeekDirection.Forward) { - return db.Seek(options, prefix, direction, (k, v) => (k, v)) - ?? db.Seek(ReadOptions.Default, prefix, direction, (k, v) => (k, v)); + return db.Seek(options, prefix, direction, (k, v) => (k, v)); } public void Put(byte[] key, byte[] value) @@ -59,12 +58,12 @@ public void Put(byte[] key, byte[] value) public bool Contains(byte[] key) { - return db.Contains(options, key) || db.Contains(ReadOptions.Default, key); + return db.Contains(options, key); } public byte[] TryGet(byte[] key) { - return db.Get(options, key) ?? db.Get(ReadOptions.Default, key); + return db.Get(options, key); } } } diff --git a/src/Plugins/RocksDBStore/Plugins/Storage/Snapshot.cs b/src/Plugins/RocksDBStore/Plugins/Storage/Snapshot.cs index 1078ea7778..7423f6ae4a 100644 --- a/src/Plugins/RocksDBStore/Plugins/Storage/Snapshot.cs +++ b/src/Plugins/RocksDBStore/Plugins/Storage/Snapshot.cs @@ -65,12 +65,12 @@ public IEnumerable<(byte[] Key, byte[] Value)> Seek(byte[] keyOrPrefix, SeekDire public bool Contains(byte[] key) { - return db.Get(key, Array.Empty(), 0, 0, readOptions: options) >= 0 || db.Get(key, Array.Empty(), 0, 0) >= 0; + return db.Get(key, Array.Empty(), 0, 0, readOptions: options) >= 0; } public byte[] TryGet(byte[] key) { - return db.Get(key, readOptions: options) ?? db.Get(key); + return db.Get(key, readOptions: options); } public void Dispose() diff --git a/tests/Neo.Plugins.Storage.Tests/StoreTest.cs b/tests/Neo.Plugins.Storage.Tests/StoreTest.cs index d75c3d75d2..c47f39013e 100644 --- a/tests/Neo.Plugins.Storage.Tests/StoreTest.cs +++ b/tests/Neo.Plugins.Storage.Tests/StoreTest.cs @@ -93,10 +93,10 @@ public void TestLevelDbSnapshot() snapshot.Commit(); - // After commit, the data shall be visible to the store and to the snapshot - CollectionAssert.AreEqual(testValue, snapshot.TryGet(testKey)); + // After commit, the data shall be visible to the store but not to the snapshot + Assert.IsNull(snapshot.TryGet(testKey)); CollectionAssert.AreEqual(testValue, store.TryGet(testKey)); - Assert.AreEqual(true, snapshot.Contains(testKey)); + Assert.AreEqual(false, snapshot.Contains(testKey)); Assert.AreEqual(true, store.Contains(testKey)); snapshot.Dispose(); @@ -109,7 +109,6 @@ public void TestLevelDbMultiSnapshot() using var store = levelDbStore.GetStore(path_leveldb); var snapshot = store.GetSnapshot(); - var snapshot2 = store.GetSnapshot(); var testKey = new byte[] { 0x01, 0x02, 0x03 }; var testValue = new byte[] { 0x04, 0x05, 0x06 }; @@ -118,8 +117,11 @@ public void TestLevelDbMultiSnapshot() snapshot.Commit(); CollectionAssert.AreEqual(testValue, store.TryGet(testKey)); - // Data saved to the leveldb from snapshot1 shall also be visible to snapshot2 + var snapshot2 = store.GetSnapshot(); + + // Data saved to the leveldb from snapshot1 shall be visible to snapshot2 but not visible to snapshot1 CollectionAssert.AreEqual(testValue, snapshot2.TryGet(testKey)); + Assert.IsNull(snapshot.TryGet(testKey)); snapshot.Dispose(); snapshot2.Dispose(); @@ -153,8 +155,9 @@ public void TestRocksDbSnapshot() var testValue = new byte[] { 0x04, 0x05, 0x06 }; snapshot.Put(testKey, testValue); - // Data saved to the leveldb snapshot shall not be visible to the store + // Data saved to the leveldb snapshot shall not be visible Assert.IsNull(snapshot.TryGet(testKey)); + Assert.IsNull(store.TryGet(testKey)); // Value is in the write batch, not visible to the store and snapshot Assert.AreEqual(false, snapshot.Contains(testKey)); @@ -162,10 +165,10 @@ public void TestRocksDbSnapshot() snapshot.Commit(); - // After commit, the data shall be visible to the store and to the snapshot - CollectionAssert.AreEqual(testValue, snapshot.TryGet(testKey)); + // After commit, the data shall be visible to the store but not to the snapshot + Assert.IsNull(snapshot.TryGet(testKey)); CollectionAssert.AreEqual(testValue, store.TryGet(testKey)); - Assert.AreEqual(true, snapshot.Contains(testKey)); + Assert.AreEqual(false, snapshot.Contains(testKey)); Assert.AreEqual(true, store.Contains(testKey)); snapshot.Dispose(); @@ -178,7 +181,6 @@ public void TestRocksDbMultiSnapshot() using var store = rocksDBStore.GetStore(path_leveldb); var snapshot = store.GetSnapshot(); - var snapshot2 = store.GetSnapshot(); var testKey = new byte[] { 0x01, 0x02, 0x03 }; var testValue = new byte[] { 0x04, 0x05, 0x06 }; @@ -187,7 +189,8 @@ public void TestRocksDbMultiSnapshot() snapshot.Commit(); CollectionAssert.AreEqual(testValue, store.TryGet(testKey)); - // Data saved to the leveldb from snapshot1 shall also be visible to snapshot2 + var snapshot2 = store.GetSnapshot(); + // Data saved to the leveldb from snapshot1 shall only be visible to snapshot2 CollectionAssert.AreEqual(testValue, snapshot2.TryGet(testKey)); snapshot.Dispose(); From 2417de1a8ebc90db57064e41b3d7732f91c1076c Mon Sep 17 00:00:00 2001 From: Jimmy Date: Sun, 7 Jul 2024 00:05:51 +0800 Subject: [PATCH 5/6] remove duplicate --- tests/Neo.Plugins.Storage.Tests/StoreTest.cs | 4 ---- 1 file changed, 4 deletions(-) diff --git a/tests/Neo.Plugins.Storage.Tests/StoreTest.cs b/tests/Neo.Plugins.Storage.Tests/StoreTest.cs index c47f39013e..83a2715f08 100644 --- a/tests/Neo.Plugins.Storage.Tests/StoreTest.cs +++ b/tests/Neo.Plugins.Storage.Tests/StoreTest.cs @@ -75,7 +75,6 @@ public void TestLevelDb() [TestMethod] public void TestLevelDbSnapshot() { - ReSetStore(); using var store = levelDbStore.GetStore(path_leveldb); var snapshot = store.GetSnapshot(); @@ -105,7 +104,6 @@ public void TestLevelDbSnapshot() [TestMethod] public void TestLevelDbMultiSnapshot() { - ReSetStore(); using var store = levelDbStore.GetStore(path_leveldb); var snapshot = store.GetSnapshot(); @@ -146,7 +144,6 @@ public void TestRocksDb() [TestMethod] public void TestRocksDbSnapshot() { - ReSetStore(); using var store = rocksDBStore.GetStore(path_leveldb); var snapshot = store.GetSnapshot(); @@ -177,7 +174,6 @@ public void TestRocksDbSnapshot() [TestMethod] public void TestRocksDbMultiSnapshot() { - ReSetStore(); using var store = rocksDBStore.GetStore(path_leveldb); var snapshot = store.GetSnapshot(); From 5b5342f126f264247bf8f568ceff3ffcc570ee9e Mon Sep 17 00:00:00 2001 From: Shargon Date: Mon, 8 Jul 2024 01:20:21 -0700 Subject: [PATCH 6/6] Remove method --- tests/Neo.Plugins.Storage.Tests/StoreTest.cs | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/tests/Neo.Plugins.Storage.Tests/StoreTest.cs b/tests/Neo.Plugins.Storage.Tests/StoreTest.cs index 83a2715f08..189e212bc2 100644 --- a/tests/Neo.Plugins.Storage.Tests/StoreTest.cs +++ b/tests/Neo.Plugins.Storage.Tests/StoreTest.cs @@ -29,13 +29,14 @@ public void OnStart() { levelDbStore ??= new LevelDBStore(); rocksDBStore ??= new RocksDBStore(); - ReSetStore(); + OnEnd(); } [TestCleanup] public void OnEnd() { - ReSetStore(); + if (Directory.Exists(path_leveldb)) Directory.Delete(path_leveldb, true); + if (Directory.Exists(path_rocksdb)) Directory.Delete(path_rocksdb, true); } [TestMethod] @@ -58,7 +59,6 @@ public void TestMemory() [TestMethod] public void TestLevelDb() { - ReSetStore(); TestPersistenceDelete(levelDbStore.GetStore(path_leveldb)); // Test all with the same store @@ -309,11 +309,5 @@ private void TestPersistenceRead(IStore store, bool shouldExist) else Assert.IsNull(ret); } } - - private void ReSetStore() - { - if (Directory.Exists(path_leveldb)) Directory.Delete(path_leveldb, true); - if (Directory.Exists(path_rocksdb)) Directory.Delete(path_rocksdb, true); - } } }