Skip to content

Commit

Permalink
Scavenge the index after scavenging the db:
Browse files Browse the repository at this point in the history 
- Test the TableIndex scavenge process.
- Test the index is scavenged at the end of a DB scavenge.
- Remove the ExistsAt check when doing an index table merge. The existsAt is still used when upgrading from a v1 index.
  • Loading branch information
@lscpike
lscpike committed May 29, 2018
1 parent e942c49 commit 724043a
Show file tree
Hide file tree
Showing 37 changed files with 1,801 additions and 166 deletions.
10 changes: 10 additions & 0 deletions src/EventStore.Core.Tests/EventStore.Core.Tests.csproj
View file
Expand Up @@ -278,6 +278,7 @@
<Compile Include="Index\IndexV2\table_index_hash_collision_when_upgrading_to_64bit.cs" />
<Compile Include="Index\IndexV2\table_index_when_merging_upgrading_to_64bit_if_entry_doesnt_exist_drops_entry_and_carries_on.cs" />
<Compile Include="Index\IndexV2\opening_a_ptable_with_more_than_32bits_of_records.cs" />
<Compile Include="Index\Scavenge\when_scavenging_an_index.cs" />
<Compile Include="Index\MemTableTests.cs" />
<Compile Include="Index\IndexV1\adding_four_items_to_empty_index_map_with_four_tables_per_level_causes_merge.cs" />
<Compile Include="Index\IndexV1\adding_two_items_to_empty_index_map_with_two_tables_per_level_causes_merge.cs" />
Expand Down Expand Up @@ -314,6 +315,15 @@
<Compile Include="Index\IndexVAny\when_opening_ptable_without_right_flag_in_header.cs" />
<Compile Include="Index\IndexV1\adding_four_items_to_empty_index_map_with_two_tables_per_level_causes_double_merge.cs" />
<Compile Include="Index\IndexV1\adding_item_to_empty_index_map.cs" />
<Compile Include="Index\Scavenge\when_scavenging_an_index_fails.cs" />
<Compile Include="Index\Scavenge\when_scavenging_an_index_is_cancelled.cs" />
<Compile Include="Index\Scavenge\when_scavenging_an_index_removes_nothing.cs" />
<Compile Include="Index\Scavenge\when_scavenging_a_table_index.cs" />
<Compile Include="Index\Scavenge\when_scavenging_a_table_index_and_another_table_is_completed_during.cs" />
<Compile Include="Index\Scavenge\when_scavenging_a_table_index_cancelled_while_scavenging_table.cs" />
<Compile Include="Index\Scavenge\when_scavenging_a_table_index_cancelled_while_waiting_for_lock.cs" />
<Compile Include="Index\Scavenge\when_scavenging_a_table_index_fails.cs" />
<Compile Include="Index\Scavenge\when_scavenging_a_v1_index.cs" />
<Compile Include="Infrastructure\RandTestQueueItem.cs" />
<Compile Include="Infrastructure\IRandTestFinishCondition.cs" />
<Compile Include="Infrastructure\IRandTestItemProcessor.cs" />
Expand Down
9 changes: 8 additions & 1 deletion src/EventStore.Core.Tests/Index/FakeIndexReader.cs
View file
Expand Up @@ -6,6 +6,13 @@ namespace EventStore.Core.Tests.Fakes
{
public class FakeIndexReader : ITransactionFileReader
{
private readonly Func<long, bool> _existsAt;

public FakeIndexReader(Func<long,bool> existsAt = null)
{
_existsAt = existsAt ?? (l => true);
}

public void Reposition(long position)
{
throw new NotImplementedException();
Expand All @@ -29,7 +36,7 @@ public RecordReadResult TryReadAt(long position)

public bool ExistsAt(long position)
{
return true;
return _existsAt(position);
}
}
}
13 changes: 9 additions & 4 deletions src/EventStore.Core.Tests/Index/IndexV1/when_merging_ptables.cs
View file
Expand Up @@ -277,9 +277,11 @@ public void merged_ptable_is_64bit()
}

[Test]
public void there_are_5_records_in_the_merged_index()
public void there_are_8_records_in_the_merged_index()
{
Assert.AreEqual(5, _newtable.Count);
// 5 from the 64 bit table (existsAt doesn't get used)
// 3 from the 32 bit table (3 even positions)
Assert.AreEqual(8, _newtable.Count);
}

[Test]
Expand Down Expand Up @@ -357,9 +359,12 @@ public void merged_ptable_is_64bit()
}

[Test]
public void there_are_7_records_in_the_merged_index()
public void there_are_10_records_in_the_merged_index()
{
Assert.AreEqual(7, _newtable.Count);
// 5 from 64 bit (existsAt not called)
// 2 from first table (2 even positions)
// 3 from last table (3 even positions)
Assert.AreEqual(10, _newtable.Count);
}

[Test]
Expand Down
19 changes: 6 additions & 13 deletions ...Store.Core.Tests/Index/IndexV1/when_merging_ptables_with_entries_to_nonexisting_record.cs
View file
Expand Up @@ -6,13 +6,11 @@

namespace EventStore.Core.Tests.Index.IndexV1
{
[TestFixture(PTableVersions.IndexV1,false)]
[TestFixture(PTableVersions.IndexV1,true)]
[TestFixture(PTableVersions.IndexV2,false)]
[TestFixture(PTableVersions.IndexV2,true)]
[TestFixture(PTableVersions.IndexV3,false)]
[TestFixture(PTableVersions.IndexV3,true)]
public class when_merging_ptables_with_entries_to_nonexisting_record: SpecificationWithDirectoryPerTestFixture
public class when_merging_ptables_with_entries_to_nonexisting_record_in_newer_index_versions: SpecificationWithDirectoryPerTestFixture
{
private readonly List<string> _files = new List<string>();
private readonly List<PTable> _tables = new List<PTable>();
Expand All @@ -21,7 +19,7 @@ public class when_merging_ptables_with_entries_to_nonexisting_record: Specificat

private bool _skipIndexVerify;

public when_merging_ptables_with_entries_to_nonexisting_record(byte version, bool skipIndexVerify){
public when_merging_ptables_with_entries_to_nonexisting_record_in_newer_index_versions(byte version, bool skipIndexVerify){
_ptableVersion = version;
_skipIndexVerify = skipIndexVerify;
}
Expand Down Expand Up @@ -57,9 +55,9 @@ public override void TestFixtureTearDown()
}

[Test]
public void there_are_only_twenty_entries_left()
public void all_entries_are_left()
{
Assert.AreEqual(20, _newtable.Count);
Assert.AreEqual(40, _newtable.Count);
}

[Test]
Expand All @@ -82,13 +80,8 @@ public void the_right_items_are_deleted()
for (int j = 0; j < 10; j++)
{
long position;
if ((i*10 + j)%2 == 0)
{
Assert.IsTrue(_newtable.TryGetOneValue((ulong)(0x010100000000 << i), j, out position));
Assert.AreEqual(i*10+j, position);
}
else
Assert.IsFalse(_newtable.TryGetOneValue((ulong)(0x010100000000 << i), j, out position));
Assert.IsTrue(_newtable.TryGetOneValue((ulong)(0x010100000000 << i), j, out position));
Assert.AreEqual(i*10+j, position);
}
}
}
Expand Down
122 changes: 116 additions & 6 deletions src/EventStore.Core.Tests/Index/IndexV4/when_merging_ptables_vx_to_v4.cs
View file
Expand Up @@ -143,8 +143,118 @@ public void none_of_the_entries_have_upgraded_hashes()
}
}

[TestFixture(PTableVersions.IndexV1,false)]
[TestFixture(PTableVersions.IndexV1,true)]
[TestFixture(PTableVersions.IndexV1, false)]
[TestFixture(PTableVersions.IndexV1, true)]
public class when_merging_to_ptable_v4_with_deleted_entries_from_v1 : SpecificationWithDirectoryPerTestFixture
{
private readonly List<string> _files = new List<string>();
private readonly List<PTable> _tables = new List<PTable>();
private IHasher hasher;
private string _newtableFile;

private PTable _newtable;
private byte _fromVersion;
private bool _skipIndexVerify;

public when_merging_to_ptable_v4_with_deleted_entries_from_v1(byte fromVersion, bool skipIndexVerify)
{
_fromVersion = fromVersion;
_skipIndexVerify = skipIndexVerify;
}

[OneTimeSetUp]
public override void TestFixtureSetUp()
{
hasher = new Murmur3AUnsafe();
base.TestFixtureSetUp();
_files.Add(GetTempFilePath());
var table = new HashListMemTable(_fromVersion, maxSize: 20);
table.Add(0x010100000000, 0, 1);
table.Add(0x010200000000, 0, 2);
table.Add(0x010300000000, 0, 3);
table.Add(0x010300000000, 1, 4);
_tables.Add(PTable.FromMemtable(table, GetTempFilePath(), skipIndexVerify: _skipIndexVerify));
table = new HashListMemTable(_fromVersion, maxSize: 20);
table.Add(0x010100000000, 2, 5);
table.Add(0x010200000000, 1, 6);
table.Add(0x010200000000, 2, 7);
table.Add(0x010400000000, 0, 8);
table.Add(0x010400000000, 1, 9);
_tables.Add(PTable.FromMemtable(table, GetTempFilePath(), skipIndexVerify: _skipIndexVerify));
table = new HashListMemTable(_fromVersion, maxSize: 20);
table.Add(0x010100000000, 1, 10);
table.Add(0x010100000000, 2, 11);
table.Add(0x010500000000, 1, 12);
table.Add(0x010500000000, 2, 13);
table.Add(0x010500000000, 3, 14);
_tables.Add(PTable.FromMemtable(table, GetTempFilePath(), skipIndexVerify: _skipIndexVerify));
_newtableFile = GetTempFilePath();
_newtable = PTable.MergeTo(_tables, _newtableFile, (streamId, hash) => hash << 32 | hasher.Hash(streamId), x => x.Position % 2 == 0, x => new Tuple<string, bool>(x.Stream.ToString(), x.Position % 2 == 0), PTableVersions.IndexV4, skipIndexVerify: _skipIndexVerify);
}

[OneTimeTearDown]
public override void TestFixtureTearDown()
{
_newtable.Dispose();
foreach (var ssTable in _tables)
{
ssTable.Dispose();
}
base.TestFixtureTearDown();
}

[Test]
public void merged_ptable_is_64bit()
{
Assert.AreEqual(PTableVersions.IndexV4, _newtable.Version);
}

[Test]
public void there_are_7_records_in_the_merged_index()
{
Assert.AreEqual(7, _newtable.Count);
}

[Test]
public void midpoints_are_cached_in_ptable_footer()
{
var numIndexEntries = 7;
var requiredMidpoints = PTable.GetRequiredMidpointCountCached(numIndexEntries, PTableVersions.IndexV4);

var newTableFileCopy = GetTempFilePath();
File.Copy(_newtableFile, newTableFileCopy);
using (var filestream = File.Open(newTableFileCopy, FileMode.Open, FileAccess.Read))
{
var footerSize = PTableFooter.GetSize(PTableVersions.IndexV4);
Assert.AreEqual(filestream.Length, PTableHeader.Size + numIndexEntries * PTable.IndexEntryV4Size + requiredMidpoints * PTable.IndexEntryV4Size + footerSize + PTable.MD5Size);
filestream.Seek(PTableHeader.Size + numIndexEntries * PTable.IndexEntryV4Size + requiredMidpoints * PTable.IndexEntryV4Size, SeekOrigin.Begin);

var ptableFooter = PTableFooter.FromStream(filestream);
Assert.AreEqual(FileType.PTableFile, ptableFooter.FileType);
Assert.AreEqual(PTableVersions.IndexV4, ptableFooter.Version);
Assert.AreEqual(requiredMidpoints, ptableFooter.NumMidpointsCached);
}
}

[Test]
public void correct_number_of_midpoints_are_loaded()
{
Assert.AreEqual(_newtable.GetMidPoints().Length, PTable.GetRequiredMidpointCountCached(7, PTableVersions.IndexV4));
}

[Test]
public void the_items_are_sorted()
{
var last = new IndexEntry(ulong.MaxValue, 0, long.MaxValue);
foreach (var item in _newtable.IterateAllInOrder())
{
Assert.IsTrue((last.Stream == item.Stream ? last.Version > item.Version : last.Stream > item.Stream) ||
((last.Stream == item.Stream && last.Version == item.Version) && last.Position > item.Position));
last = item;
}
}
}

[TestFixture(PTableVersions.IndexV2,false)]
[TestFixture(PTableVersions.IndexV2,true)]
[TestFixture(PTableVersions.IndexV3,false)]
Expand Down Expand Up @@ -215,15 +325,15 @@ public void merged_ptable_is_64bit()
}

[Test]
public void there_are_7_records_in_the_merged_index()
public void there_are_14_records_in_the_merged_index()
{
Assert.AreEqual(7, _newtable.Count);
Assert.AreEqual(14, _newtable.Count);
}

[Test]
public void midpoints_are_cached_in_ptable_footer()
{
var numIndexEntries = 7;
var numIndexEntries = 14;
var requiredMidpoints = PTable.GetRequiredMidpointCountCached(numIndexEntries,PTableVersions.IndexV4);

var newTableFileCopy = GetTempFilePath();
Expand All @@ -244,7 +354,7 @@ public void midpoints_are_cached_in_ptable_footer()
[Test]
public void correct_number_of_midpoints_are_loaded()
{
Assert.AreEqual(_newtable.GetMidPoints().Length, PTable.GetRequiredMidpointCountCached(7,PTableVersions.IndexV4));
Assert.AreEqual(_newtable.GetMidPoints().Length, PTable.GetRequiredMidpointCountCached(14,PTableVersions.IndexV4));
}

[Test]
Expand Down
2 changes: 1 addition & 1 deletion ...Store.Core.Tests/Index/IndexV4/when_merging_ptables_with_entries_to_nonexisting_record.cs
View file
Expand Up @@ -5,7 +5,7 @@ namespace EventStore.Core.Tests.Index.IndexV4
{
[TestFixture(PTableVersions.IndexV4,false)]
[TestFixture(PTableVersions.IndexV4,true)]
public class when_merging_ptables_with_entries_to_nonexisting_record: IndexV1.when_merging_ptables_with_entries_to_nonexisting_record
public class when_merging_ptables_with_entries_to_nonexisting_record: IndexV1.when_merging_ptables_with_entries_to_nonexisting_record_in_newer_index_versions
{
public when_merging_ptables_with_entries_to_nonexisting_record(byte version, bool skipIndexVerify):base(version,skipIndexVerify)
{
Expand Down

0 comments on commit 724043a

Please sign in to comment.