From 123d46e6b164710ff745e9bd2c3c91925d4823fa Mon Sep 17 00:00:00 2001 From: Anton Persson Date: Thu, 24 Nov 2016 17:01:51 +0100 Subject: [PATCH] Testing for create new version of node on write if stable and PointerChecking update Add checkSiblingPointer to PointerChecking Use checkSiblingPointer and checkChildPointer whenever a child or sibling pointer is read. Testing for PointerChecking PageAwareByteArrayCursor now support openLinkedCursor. --- .../index/gbptree/ByteArrayPageCursor.java | 2 +- .../java/org/neo4j/index/gbptree/IdSpace.java | 14 +- .../index/gbptree/InternalTreeLogic.java | 9 + .../neo4j/index/gbptree/PointerChecking.java | 13 + .../index/gbptree/InternalTreeLogicTest.java | 367 +++++++++++++++++- .../gbptree/PageAwareByteArrayCursor.java | 46 ++- .../index/gbptree/PointerCheckingTest.java | 105 ++++- .../neo4j/index/gbptree/SeekCursorTest.java | 1 - 8 files changed, 511 insertions(+), 46 deletions(-) diff --git a/community/index/src/main/java/org/neo4j/index/gbptree/ByteArrayPageCursor.java b/community/index/src/main/java/org/neo4j/index/gbptree/ByteArrayPageCursor.java index fbfd312e91614..a548980feea64 100644 --- a/community/index/src/main/java/org/neo4j/index/gbptree/ByteArrayPageCursor.java +++ b/community/index/src/main/java/org/neo4j/index/gbptree/ByteArrayPageCursor.java @@ -210,7 +210,7 @@ public long getCurrentPageId() @Override public int getCurrentPageSize() { - throw new UnsupportedOperationException(); + return buffer.capacity(); } @Override diff --git a/community/index/src/main/java/org/neo4j/index/gbptree/IdSpace.java b/community/index/src/main/java/org/neo4j/index/gbptree/IdSpace.java index 73cec73a3f7cf..5a3a5e5be35d2 100644 --- a/community/index/src/main/java/org/neo4j/index/gbptree/IdSpace.java +++ b/community/index/src/main/java/org/neo4j/index/gbptree/IdSpace.java @@ -27,8 +27,20 @@ public class IdSpace */ static final long META_PAGE_ID = 0L; + /** + * State page with IDs such as free-list, highId, rootId and more. There are two such pages alternating + * between checkpoints, this is the first. + */ + static final long STATE_PAGE_A = 1L; + + /** + * State page with IDs such as free-list, highId, rootId and more. There are two such pages alternating + * between checkpoints, this is the second. + */ + static final long STATE_PAGE_B = 2L; + /** * Min value allowed as tree node id. */ - static final long MIN_TREE_NODE_ID = 1L; + static final long MIN_TREE_NODE_ID = 3L; } diff --git a/community/index/src/main/java/org/neo4j/index/gbptree/InternalTreeLogic.java b/community/index/src/main/java/org/neo4j/index/gbptree/InternalTreeLogic.java index 5b9ec69092fc7..e222bd01c1e5c 100644 --- a/community/index/src/main/java/org/neo4j/index/gbptree/InternalTreeLogic.java +++ b/community/index/src/main/java/org/neo4j/index/gbptree/InternalTreeLogic.java @@ -231,6 +231,7 @@ private void splitInternal( PageCursor cursor, StructurePropagation structu { long current = cursor.getCurrentPageId(); long oldRight = bTreeNode.rightSibling( cursor, stableGeneration, unstableGeneration ); + PointerChecking.checkSiblingPointer( oldRight ); long newRight = idProvider.acquireNewId(); // Find position to insert new key @@ -386,6 +387,7 @@ private void splitLeaf( PageCursor cursor, StructurePropagation structurePr long current = cursor.getCurrentPageId(); long oldRight = bTreeNode.rightSibling( cursor, stableGeneration, unstableGeneration ); + PointerChecking.checkSiblingPointer( oldRight ); long newRight = idProvider.acquireNewId(); // BALANCE KEYS AND VALUES @@ -529,6 +531,7 @@ VALUE remove( PageCursor cursor, StructurePropagation structurePropagation, long currentId = cursor.getCurrentPageId(); long childId = bTreeNode.childAt( cursor, pos, stableGeneration, unstableGeneration ); + PointerChecking.checkChildPointer( childId ); bTreeNode.goTo( cursor, childId, stableGeneration, unstableGeneration ); VALUE result = remove( cursor, structurePropagation, key, into, stableGeneration, unstableGeneration ); @@ -619,6 +622,10 @@ private void createUnstableVersionIfNeeded( PageCursor cursor, StructurePropagat long newGenId = idProvider.acquireNewId(); try ( PageCursor newGenCursor = cursor.openLinkedCursor( newGenId ) ) { + if ( !newGenCursor.next() ) + { + throw new IllegalStateException( "Could not go to new node " + newGenId ); + } cursor.copyTo( 0, newGenCursor, 0, cursor.getCurrentPageSize() ); bTreeNode.setGen( newGenCursor, unstableGeneration ); } @@ -640,7 +647,9 @@ private void createUnstableVersionIfNeeded( PageCursor cursor, StructurePropagat // v v v // (leftSiblingOfStableNode) -[rightSibling]-> (newUnstableNode) <-[leftSibling]- (rightSiblingOfStableNode) long leftSibling = bTreeNode.leftSibling( cursor, stableGeneration, unstableGeneration ); + PointerChecking.checkSiblingPointer( leftSibling ); long rightSibling = bTreeNode.rightSibling( cursor, stableGeneration, unstableGeneration ); + PointerChecking.checkSiblingPointer( rightSibling ); if ( leftSibling != TreeNode.NO_NODE_FLAG ) { bTreeNode.goTo( cursor, leftSibling, stableGeneration, unstableGeneration ); diff --git a/community/index/src/main/java/org/neo4j/index/gbptree/PointerChecking.java b/community/index/src/main/java/org/neo4j/index/gbptree/PointerChecking.java index 1ed7ab0783480..fbb9995f3ddf7 100644 --- a/community/index/src/main/java/org/neo4j/index/gbptree/PointerChecking.java +++ b/community/index/src/main/java/org/neo4j/index/gbptree/PointerChecking.java @@ -42,4 +42,17 @@ static void checkChildPointer( long result ) IdSpace.MIN_TREE_NODE_ID ); } } + + static void checkSiblingPointer( long result ) + { + if ( !GenSafePointerPair.isSuccess( result ) ) + { + throw new IllegalStateException( GenSafePointerPair.failureDescription( result ) ); + } + if ( result < IdSpace.MIN_TREE_NODE_ID && result != TreeNode.NO_NODE_FLAG ) + { + throw new IllegalStateException( "Pointer to id " + result + " not allowed. Minimum node id allowed is " + + IdSpace.MIN_TREE_NODE_ID ); + } + } } diff --git a/community/index/src/test/java/org/neo4j/index/gbptree/InternalTreeLogicTest.java b/community/index/src/test/java/org/neo4j/index/gbptree/InternalTreeLogicTest.java index 56ba705cf4dc1..120af02634254 100644 --- a/community/index/src/test/java/org/neo4j/index/gbptree/InternalTreeLogicTest.java +++ b/community/index/src/test/java/org/neo4j/index/gbptree/InternalTreeLogicTest.java @@ -33,6 +33,7 @@ import static org.hamcrest.CoreMatchers.is; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotEquals; import static org.junit.Assert.assertNull; import static org.junit.Assert.assertThat; import static org.junit.Assert.assertTrue; @@ -46,8 +47,9 @@ public class InternalTreeLogicTest return base; }; - private static final int STABLE_GENERATION = 1; - private static final int UNSTABLE_GENERATION = 2; + private static final long OLD_STABLE_GENERATION = 1; + private static final long STABLE_GENERATION = 2; + private static final long UNSTABLE_GENERATION = 3; private final int pageSize = 256; private final SimpleIdProvider id = new SimpleIdProvider(); @@ -72,16 +74,15 @@ public class InternalTreeLogicTest public void setUp() throws IOException { id.reset(); - cursor.initialize(); long newId = id.acquireNewId(); cursor.next( newId ); - node.initializeLeaf( cursor, STABLE_GENERATION, UNSTABLE_GENERATION ); } @Test public void modifierMustInsertAtFirstPositionInEmptyLeaf() throws Exception { // given + node.initializeLeaf( cursor, STABLE_GENERATION, UNSTABLE_GENERATION ); long key = 1L; long value = 1L; assertThat( node.keyCount( cursor ), is( 0 ) ); @@ -98,6 +99,7 @@ public void modifierMustInsertAtFirstPositionInEmptyLeaf() throws Exception @Test public void modifierMustSortCorrectlyOnInsertFirstInLeaf() throws Exception { + node.initializeLeaf( cursor, STABLE_GENERATION, UNSTABLE_GENERATION ); for ( int i = 0; i < maxKeyCount; i++ ) { // given @@ -113,6 +115,7 @@ public void modifierMustSortCorrectlyOnInsertFirstInLeaf() throws Exception @Test public void modifierMustSortCorrectlyOnInsertLastInLeaf() throws Exception { + node.initializeLeaf( cursor, STABLE_GENERATION, UNSTABLE_GENERATION ); for ( int i = 0; i < maxKeyCount; i++ ) { // given @@ -127,6 +130,7 @@ public void modifierMustSortCorrectlyOnInsertLastInLeaf() throws Exception @Test public void modifierMustSortCorrectlyOnInsertInMiddleOfLeaf() throws Exception { + node.initializeLeaf( cursor, STABLE_GENERATION, UNSTABLE_GENERATION ); for ( int i = 0; i < maxKeyCount; i++ ) { // given @@ -142,6 +146,7 @@ public void modifierMustSortCorrectlyOnInsertInMiddleOfLeaf() throws Exception public void modifierMustSplitWhenInsertingMiddleOfFullLeaf() throws Exception { // given + node.initializeLeaf( cursor, STABLE_GENERATION, UNSTABLE_GENERATION ); for ( int i = 0; i < maxKeyCount; i++ ) { long key = i % 2 == 0 ? i : maxKeyCount * 2 - i; @@ -158,6 +163,7 @@ public void modifierMustSplitWhenInsertingMiddleOfFullLeaf() throws Exception public void modifierMustSplitWhenInsertingLastInFullLeaf() throws Exception { // given + node.initializeLeaf( cursor, STABLE_GENERATION, UNSTABLE_GENERATION ); long key = 0; while ( key < maxKeyCount ) { @@ -175,6 +181,7 @@ public void modifierMustSplitWhenInsertingLastInFullLeaf() throws Exception public void modifierMustSplitWhenInsertingFirstInFullLeaf() throws Exception { // given + node.initializeLeaf( cursor, STABLE_GENERATION, UNSTABLE_GENERATION ); for ( int i = 0; i < maxKeyCount; i++ ) { long key = i + 1; @@ -191,6 +198,7 @@ public void modifierMustSplitWhenInsertingFirstInFullLeaf() throws Exception public void modifierMustLeaveCursorOnSamePageAfterSplitInLeaf() throws Exception { // given + node.initializeLeaf( cursor, STABLE_GENERATION, UNSTABLE_GENERATION ); long pageId = cursor.getCurrentPageId(); long key = 0; while ( key < maxKeyCount ) @@ -212,6 +220,7 @@ public void modifierMustLeaveCursorOnSamePageAfterSplitInLeaf() throws Exception public void modifierMustUpdatePointersInSiblingsToSplit() throws Exception { // given + node.initializeLeaf( cursor, STABLE_GENERATION, UNSTABLE_GENERATION ); long someLargeNumber = maxKeyCount * 1000; long i = 0; while ( i < maxKeyCount ) @@ -255,6 +264,7 @@ public void modifierMustUpdatePointersInSiblingsToSplit() throws Exception public void modifierMustRemoveFirstInEmptyLeaf() throws Exception { // given + node.initializeLeaf( cursor, STABLE_GENERATION, UNSTABLE_GENERATION ); long key = 1L; long value = 1L; insert( key, value ); @@ -270,6 +280,7 @@ public void modifierMustRemoveFirstInEmptyLeaf() throws Exception public void modifierMustRemoveFirstInFullLeaf() throws Exception { // given + node.initializeLeaf( cursor, STABLE_GENERATION, UNSTABLE_GENERATION ); for ( int i = 0; i < maxKeyCount; i++ ) { insert( i, i ); @@ -290,6 +301,7 @@ public void modifierMustRemoveFirstInFullLeaf() throws Exception public void modifierMustRemoveInMiddleInFullLeaf() throws Exception { // given + node.initializeLeaf( cursor, STABLE_GENERATION, UNSTABLE_GENERATION ); int middle = maxKeyCount / 2; for ( int i = 0; i < maxKeyCount; i++ ) { @@ -313,6 +325,7 @@ public void modifierMustRemoveInMiddleInFullLeaf() throws Exception public void modifierMustRemoveLastInFullLeaf() throws Exception { // given + node.initializeLeaf( cursor, STABLE_GENERATION, UNSTABLE_GENERATION ); for ( int i = 0; i < maxKeyCount; i++ ) { insert( i, i ); @@ -334,6 +347,7 @@ public void modifierMustRemoveLastInFullLeaf() throws Exception public void modifierMustRemoveFromLeftChild() throws Exception { // given + node.initializeLeaf( cursor, STABLE_GENERATION, UNSTABLE_GENERATION ); for ( int i = 0; !structurePropagation.hasSplit; i++ ) { insert( i, i ); @@ -355,6 +369,7 @@ public void modifierMustRemoveFromLeftChild() throws Exception public void modifierMustRemoveFromRightChildButNotFromInternalWithHitOnInternalSearch() throws Exception { // given + node.initializeLeaf( cursor, STABLE_GENERATION, UNSTABLE_GENERATION ); for ( int i = 0; !structurePropagation.hasSplit; i++ ) { insert( i, i ); @@ -388,6 +403,7 @@ public void modifierMustRemoveFromRightChildButNotFromInternalWithHitOnInternalS public void modifierMustLeaveCursorOnInitialPageAfterRemove() throws Exception { // given + node.initializeLeaf( cursor, STABLE_GENERATION, UNSTABLE_GENERATION ); for ( int i = 0; !structurePropagation.hasSplit; i++ ) { insert( i, i ); @@ -406,6 +422,7 @@ public void modifierMustLeaveCursorOnInitialPageAfterRemove() throws Exception public void modifierMustNotRemoveWhenKeyDoesNotExist() throws Exception { // given + node.initializeLeaf( cursor, STABLE_GENERATION, UNSTABLE_GENERATION ); for ( int i = 0; i < maxKeyCount; i++ ) { insert( i, i ); @@ -427,6 +444,7 @@ public void modifierMustNotRemoveWhenKeyDoesNotExist() throws Exception public void modifierMustNotRemoveWhenKeyOnlyExistInInternal() throws Exception { // given + node.initializeLeaf( cursor, STABLE_GENERATION, UNSTABLE_GENERATION ); for ( int i = 0; !structurePropagation.hasSplit; i++ ) { insert( i, i ); @@ -466,6 +484,7 @@ public void modifierMustNotRemoveWhenKeyOnlyExistInInternal() throws Exception @Test public void modifierMustProduceConsistentTreeWithRandomInserts() throws Exception { + node.initializeLeaf( cursor, STABLE_GENERATION, UNSTABLE_GENERATION ); int numberOfEntries = 100_000; for ( int i = 0; i < numberOfEntries; i++ ) { @@ -487,6 +506,7 @@ public void modifierMustProduceConsistentTreeWithRandomInserts() throws Exceptio public void modifierMustOverwriteWithOverwriteMerger() throws Exception { // given + node.initializeLeaf( cursor, STABLE_GENERATION, UNSTABLE_GENERATION ); long key = random.nextLong(); long firstValue = random.nextLong(); insert( key, firstValue ); @@ -504,6 +524,7 @@ public void modifierMustOverwriteWithOverwriteMerger() throws Exception public void modifierMustKeepExistingWithKeepExistingMerger() throws Exception { // given + node.initializeLeaf( cursor, STABLE_GENERATION, UNSTABLE_GENERATION ); long key = random.nextLong(); long firstValue = random.nextLong(); insert( key, firstValue, ValueMergers.keepExisting() ); @@ -525,6 +546,7 @@ public void modifierMustKeepExistingWithKeepExistingMerger() throws Exception public void shouldMergeValueInRootLeaf() throws Exception { // GIVEN + node.initializeLeaf( cursor, STABLE_GENERATION, UNSTABLE_GENERATION ); long key = 10; long baseValue = 100; insert( key, baseValue ); @@ -546,6 +568,7 @@ public void shouldMergeValueInRootLeaf() throws Exception public void shouldMergeValueInLeafLeftOfParentKey() throws Exception { // GIVEN + node.initializeLeaf( cursor, STABLE_GENERATION, UNSTABLE_GENERATION ); for ( int i = 0; !structurePropagation.hasSplit; i++ ) { insert( i, i ); @@ -572,6 +595,7 @@ public void shouldMergeValueInLeafLeftOfParentKey() throws Exception public void shouldMergeValueInLeafAtParentKey() throws Exception { // GIVEN + node.initializeLeaf( cursor, STABLE_GENERATION, UNSTABLE_GENERATION ); for ( int i = 0; !structurePropagation.hasSplit; i++ ) { insert( i, i ); @@ -598,6 +622,7 @@ public void shouldMergeValueInLeafAtParentKey() throws Exception public void shouldMergeValueInLeafBetweenTwoParentKeys() throws Exception { // GIVEN + node.initializeLeaf( cursor, STABLE_GENERATION, UNSTABLE_GENERATION ); long rootId = -1; long middle = -1; long firstSplitPrimKey = -1; @@ -628,6 +653,301 @@ public void shouldMergeValueInLeafBetweenTwoParentKeys() throws Exception assertEquals( baseValue + toAdd, valueAt( pos ).longValue() ); } + @Test + public void shouldCreateNewVersionWhenInsertInStableRootAsLeaf() throws Exception + { + // GIVEN root + node.initializeLeaf( cursor, OLD_STABLE_GENERATION, STABLE_GENERATION ); + long oldGenId = cursor.getCurrentPageId(); + + // WHEN root -[newGen]-> newGen root + insert( 1L, 1L, STABLE_GENERATION, UNSTABLE_GENERATION ); + long newGenId = cursor.getCurrentPageId(); + + // THEN + assertTrue( structurePropagation.hasNewGen ); + assertEquals( newGenId, structurePropagation.left ); + assertNotEquals( oldGenId, newGenId ); + assertEquals( 1, node.keyCount( cursor ) ); + + node.goTo( cursor, oldGenId, STABLE_GENERATION, UNSTABLE_GENERATION ); + assertEquals( newGenId, node.newGen( cursor, STABLE_GENERATION, UNSTABLE_GENERATION ) ); + assertEquals( 0, node.keyCount( cursor ) ); + } + + @Test + public void shouldCreateNewVersionWhenRemoveInStableRootAsLeaf() throws Exception + { + // GIVEN root + node.initializeLeaf( cursor, OLD_STABLE_GENERATION, STABLE_GENERATION ); + long key = 1L; + long value = 10L; + insert( key, value, OLD_STABLE_GENERATION, STABLE_GENERATION ); + long oldGenId = cursor.getCurrentPageId(); + + // WHEN root -[newGen]-> newGen root + remove( key, readValue, STABLE_GENERATION, UNSTABLE_GENERATION ); + long newGenId = cursor.getCurrentPageId(); + + // THEN + assertTrue( structurePropagation.hasNewGen ); + assertEquals( newGenId, structurePropagation.left ); + assertNotEquals( oldGenId, newGenId ); + assertEquals( 0, node.keyCount( cursor ) ); + + node.goTo( cursor, oldGenId, STABLE_GENERATION, UNSTABLE_GENERATION ); + assertEquals( newGenId, node.newGen( cursor, STABLE_GENERATION, UNSTABLE_GENERATION ) ); + assertEquals( 1, node.keyCount( cursor ) ); + } + + @Test + public void shouldCreateNewVersionWhenInsertInStableLeaf() throws Exception + { + // GIVEN: + // ------root------- + // / | \ + // v v v + // left <--> middle <--> right + node.initializeLeaf( cursor, OLD_STABLE_GENERATION, STABLE_GENERATION ); + long targetLastId = id.lastId() + 3; // 2 splits and 1 new allocated root + long i = 0; + for ( ; id.lastId() < targetLastId; i++ ) + { + insert( i, i, OLD_STABLE_GENERATION, STABLE_GENERATION ); + if ( structurePropagation.hasSplit ) + { + newRootFromSplit( structurePropagation, OLD_STABLE_GENERATION, STABLE_GENERATION ); + } + } + assertEquals( 2, node.keyCount( cursor ) ); + long leftChild = node.childAt( cursor, 0, STABLE_GENERATION, UNSTABLE_GENERATION ); + long middleChild = node.childAt( cursor, 1, STABLE_GENERATION, UNSTABLE_GENERATION ); + long rightChild = node.childAt( cursor, 2, STABLE_GENERATION, UNSTABLE_GENERATION ); + assertSiblings( leftChild, middleChild, rightChild ); + + // WHEN + long root = cursor.getCurrentPageId(); + long middleKey = i / 2; // Should be located in middle leaf + long newValue = middleKey * 100; + insert( middleKey, newValue, STABLE_GENERATION, UNSTABLE_GENERATION ); + cursor.next( 5 ); + cursor.next( root ); + + // THEN + // root have new middle child + long expectedNewMiddleChild = targetLastId + 1; + assertEquals( expectedNewMiddleChild, id.lastId() ); + long newMiddleChild = node.childAt( cursor, 1, STABLE_GENERATION, UNSTABLE_GENERATION ); + assertEquals( expectedNewMiddleChild, newMiddleChild ); + + // old middle child has new gen + cursor.next( middleChild ); + assertEquals( newMiddleChild, node.newGen( cursor, STABLE_GENERATION, UNSTABLE_GENERATION ) ); + + // old middle child has seen no change + assertKeyAssociatedWithValue( middleKey, middleKey ); + + // new middle child has seen change + cursor.next( newMiddleChild ); + assertKeyAssociatedWithValue( middleKey, newValue ); + + // sibling pointers updated + assertSiblings( leftChild, newMiddleChild, rightChild ); + } + + @Test + public void shouldCreateNewVersionWhenRemoveInStableLeaf() throws Exception + { + // GIVEN: + // ------root------- + // / | \ + // v v v + // left <--> middle <--> right + node.initializeLeaf( cursor, OLD_STABLE_GENERATION, STABLE_GENERATION ); + long targetLastId = id.lastId() + 3; // 2 splits and 1 new allocated root + long i = 0; + for ( ; id.lastId() < targetLastId; i++ ) + { + insert( i, i, OLD_STABLE_GENERATION, STABLE_GENERATION ); + if ( structurePropagation.hasSplit ) + { + newRootFromSplit( structurePropagation, OLD_STABLE_GENERATION, STABLE_GENERATION ); + } + } + assertEquals( 2, node.keyCount( cursor ) ); + long leftChild = node.childAt( cursor, 0, STABLE_GENERATION, UNSTABLE_GENERATION ); + long middleChild = node.childAt( cursor, 1, STABLE_GENERATION, UNSTABLE_GENERATION ); + long rightChild = node.childAt( cursor, 2, STABLE_GENERATION, UNSTABLE_GENERATION ); + assertSiblings( leftChild, middleChild, rightChild ); + + // WHEN + long root = cursor.getCurrentPageId(); + long middleKey = i / 2; // Should be located in middle leaf + remove( middleKey, insertValue, STABLE_GENERATION, UNSTABLE_GENERATION ); + cursor.next( 5 ); + cursor.next( root ); + + // THEN + // root have new middle child + long expectedNewMiddleChild = targetLastId + 1; + assertEquals( expectedNewMiddleChild, id.lastId() ); + long newMiddleChild = node.childAt( cursor, 1, STABLE_GENERATION, UNSTABLE_GENERATION ); + assertEquals( expectedNewMiddleChild, newMiddleChild ); + + // old middle child has new gen + cursor.next( middleChild ); + assertEquals( newMiddleChild, node.newGen( cursor, STABLE_GENERATION, UNSTABLE_GENERATION ) ); + + // old middle child has seen no change + assertKeyAssociatedWithValue( middleKey, middleKey ); + + // new middle child has seen change + cursor.next( newMiddleChild ); + assertKeyNotFound( middleKey ); + + // sibling pointers updated + assertSiblings( leftChild, newMiddleChild, rightChild ); + } + + @Test + public void shouldCreateNewVersionWhenInsertInStableRootAsInternal() throws Exception + { + // GIVEN: + // root + // ---- ---- + // / \ + // v v + // left <-------> right + node.initializeLeaf( cursor, OLD_STABLE_GENERATION, STABLE_GENERATION ); + long i = 0; + int countToProduceAboveImageAndFullRight = + maxKeyCount /*will split root leaf into two half left/right*/ + maxKeyCount / 2; + for ( ; i < countToProduceAboveImageAndFullRight; i++ ) + { + insert( i, i, OLD_STABLE_GENERATION, STABLE_GENERATION ); + if ( structurePropagation.hasSplit ) + { + newRootFromSplit( structurePropagation, OLD_STABLE_GENERATION, STABLE_GENERATION ); + } + } + long root = cursor.getCurrentPageId(); + assertEquals( 1, node.keyCount( cursor ) ); + long leftChild = node.childAt( cursor, 0, STABLE_GENERATION, UNSTABLE_GENERATION ); + long rightChild = node.childAt( cursor, 1, STABLE_GENERATION, UNSTABLE_GENERATION ); + assertSiblings( leftChild, rightChild, TreeNode.NO_NODE_FLAG ); + + // WHEN + // root(newGen) + // ---- | --------------- + // / | \ + // v v v + // left <-> right(newGen) <--> farRight + insert( i, i, STABLE_GENERATION, UNSTABLE_GENERATION ); + assertTrue( structurePropagation.hasNewGen ); + long newRoot = cursor.getCurrentPageId(); + leftChild = node.childAt( cursor, 0, STABLE_GENERATION, UNSTABLE_GENERATION ); + rightChild = node.childAt( cursor, 1, STABLE_GENERATION, UNSTABLE_GENERATION ); + + // THEN + // siblings are correct + long farRightChild = node.childAt( cursor, 2, STABLE_GENERATION, UNSTABLE_GENERATION ); + assertSiblings( leftChild, rightChild, farRightChild ); + + // old root points to new gen root + cursor.next( root ); + assertEquals( newRoot, node.newGen( cursor, STABLE_GENERATION, UNSTABLE_GENERATION ) ); + } + + @Test + public void shouldCreateNewVersionWhenInsertInStableInternal() throws Exception + { + // GIVEN: + node.initializeLeaf( cursor, OLD_STABLE_GENERATION, STABLE_GENERATION ); + int rootAllocations = 0; + for ( int i = 0; rootAllocations < 2; i++ ) + { + long keyAndValue = i * maxKeyCount; + insert( keyAndValue, keyAndValue, OLD_STABLE_GENERATION, STABLE_GENERATION ); + if ( structurePropagation.hasSplit ) + { + newRootFromSplit( structurePropagation, OLD_STABLE_GENERATION, STABLE_GENERATION ); + rootAllocations++; + } + } + long root = cursor.getCurrentPageId(); + assertEquals( 1, node.keyCount( cursor ) ); + long leftInternal = node.childAt( cursor, 0, STABLE_GENERATION, UNSTABLE_GENERATION ); + long rightInternal = node.childAt( cursor, 1, STABLE_GENERATION, UNSTABLE_GENERATION ); + assertSiblings( leftInternal, rightInternal, TreeNode.NO_NODE_FLAG ); + cursor.next( leftInternal ); + int leftInternalKeyCount = node.keyCount( cursor ); + assertTrue( node.isInternal( cursor ) ); + long leftLeaf = node.childAt( cursor, 0, STABLE_GENERATION, UNSTABLE_GENERATION ); + cursor.next( leftLeaf ); + long firstKeyInLeaf = node.keyAt( cursor, readKey, 0 ).longValue(); + cursor.next( root ); + + // WHEN + long targetLastId = id.lastId() + 3; /*one for newGen in leaf, one for split leaf, one for newGen in internal*/ + for ( int i = 0; id.lastId() < targetLastId; i++ ) + { + insert( firstKeyInLeaf + i, firstKeyInLeaf + i, STABLE_GENERATION, UNSTABLE_GENERATION ); + assertFalse( structurePropagation.hasSplit ); // there should be no root split + } + + // THEN + // root hasn't been split further + assertEquals( root, cursor.getCurrentPageId() ); + + // there's a new generation of left internal w/ one more key in + long newGenLeftInternal = id.lastId(); + assertEquals( newGenLeftInternal, node.childAt( cursor, 0, STABLE_GENERATION, UNSTABLE_GENERATION ) ); + cursor.next( newGenLeftInternal ); + int newGenLeftInternalKeyCount = node.keyCount( cursor ); + assertEquals( leftInternalKeyCount + 1, newGenLeftInternalKeyCount ); + + // and left internal points to the new gen + cursor.next( leftInternal ); + assertEquals( newGenLeftInternal, node.newGen( cursor, STABLE_GENERATION, UNSTABLE_GENERATION ) ); + assertSiblings( newGenLeftInternal, rightInternal, TreeNode.NO_NODE_FLAG ); + } + + private void assertKeyAssociatedWithValue( long key, long expectedValue ) + { + insertKey.setValue( key ); + int search = KeySearch.search( cursor, node, insertKey, readKey, node.keyCount( cursor ) ); + assertTrue( KeySearch.isHit( search ) ); + int keyPos = KeySearch.positionOf( search ); + node.valueAt( cursor, readValue, keyPos ); + assertEquals( expectedValue, readValue.longValue() ); + } + + private void assertKeyNotFound( long key ) + { + insertKey.setValue( key ); + int search = KeySearch.search( cursor, node, insertKey, readKey, node.keyCount( cursor ) ); + assertFalse( KeySearch.isHit( search ) ); + } + + private void assertSiblings( long left, long middle, long right ) throws IOException + { + long origin = cursor.getCurrentPageId(); + cursor.next( middle ); + assertEquals( right, node.rightSibling( cursor, STABLE_GENERATION, UNSTABLE_GENERATION ) ); + assertEquals( left, node.leftSibling( cursor, STABLE_GENERATION, UNSTABLE_GENERATION ) ); + if ( left != TreeNode.NO_NODE_FLAG ) + { + cursor.next( left ); + assertEquals( middle, node.rightSibling( cursor, STABLE_GENERATION, UNSTABLE_GENERATION ) ); + } + if ( right != TreeNode.NO_NODE_FLAG ) + { + cursor.next( right ); + assertEquals( middle, node.leftSibling( cursor, STABLE_GENERATION, UNSTABLE_GENERATION ) ); + } + cursor.next( origin ); + } + // KEEP even if unused private void printTree() throws IOException { @@ -640,15 +960,22 @@ private MutableLong key( long key ) } private long newRootFromSplit( StructurePropagation split ) throws IOException + { + return newRootFromSplit( split, STABLE_GENERATION, UNSTABLE_GENERATION ); + } + + private long newRootFromSplit( StructurePropagation split, + long stableGeneration, long unstableGeneration ) throws IOException { assertTrue( split.hasSplit ); long rootId = id.acquireNewId(); cursor.next( rootId ); - node.initializeInternal( cursor, STABLE_GENERATION, UNSTABLE_GENERATION ); + node.initializeInternal( cursor, stableGeneration, unstableGeneration ); node.insertKeyAt( cursor, split.primKey, 0, 0, tmp ); node.setKeyCount( cursor, 1 ); - node.setChildAt( cursor, split.left, 0, STABLE_GENERATION, UNSTABLE_GENERATION ); - node.setChildAt( cursor, split.right, 1, STABLE_GENERATION, UNSTABLE_GENERATION ); + node.setChildAt( cursor, split.left, 0, stableGeneration, unstableGeneration ); + node.setChildAt( cursor, split.right, 1, stableGeneration, unstableGeneration ); + split.hasSplit = false; return rootId; } @@ -681,20 +1008,37 @@ private void insert( long key, long value ) throws IOException insert( key, value, overwrite() ); } + private void insert( long key, long value, long stableGeneration, long unstableGeneration ) throws IOException + { + insert( key, value, stableGeneration, unstableGeneration, overwrite() ); + } + private void insert( long key, long value, ValueMerger valueMerger ) throws IOException + { + insert( key, value, STABLE_GENERATION, UNSTABLE_GENERATION, valueMerger ); + } + + private void insert( long key, long value, long stableGeneration, long unstableGeneration, + ValueMerger valueMerger ) throws IOException { structurePropagation.hasSplit = false; structurePropagation.hasNewGen = false; insertKey.setValue( key ); insertValue.setValue( value ); treeLogic.insert( cursor, structurePropagation, insertKey, insertValue, valueMerger, DEFAULTS, - STABLE_GENERATION, UNSTABLE_GENERATION ); + stableGeneration, unstableGeneration ); } private MutableLong remove( long key, MutableLong into ) throws IOException + { + return remove( key, into, STABLE_GENERATION, UNSTABLE_GENERATION ); + } + + private MutableLong remove( long key, MutableLong into, long stableGeneration, long unstableGeneration ) + throws IOException { insertKey.setValue( key ); - return treeLogic.remove( cursor, structurePropagation, insertKey, into, STABLE_GENERATION, UNSTABLE_GENERATION ); + return treeLogic.remove( cursor, structurePropagation, insertKey, into, stableGeneration, unstableGeneration ); } private static class SimpleIdProvider implements IdProvider @@ -713,6 +1057,11 @@ public long acquireNewId() return lastId; } + long lastId() + { + return lastId; + } + private void reset() { lastId = IdSpace.MIN_TREE_NODE_ID - 1; diff --git a/community/index/src/test/java/org/neo4j/index/gbptree/PageAwareByteArrayCursor.java b/community/index/src/test/java/org/neo4j/index/gbptree/PageAwareByteArrayCursor.java index 774e027b3358a..3534a6982fa6e 100644 --- a/community/index/src/test/java/org/neo4j/index/gbptree/PageAwareByteArrayCursor.java +++ b/community/index/src/test/java/org/neo4j/index/gbptree/PageAwareByteArrayCursor.java @@ -32,17 +32,27 @@ class PageAwareByteArrayCursor extends PageCursor { private final int pageSize; + private final List pages; + private PageCursor current; private long currentPageId = UNBOUND_PAGE_ID; - private List pages = new ArrayList<>(); + private long nextPageId; + private PageCursor linkedCursor; PageAwareByteArrayCursor( int pageSize ) { + this( new ArrayList<>(), pageSize, 0 ); + } + + private PageAwareByteArrayCursor( List pages, int pageSize, long nextPageId ) + { + this.pages = pages; this.pageSize = pageSize; + this.nextPageId = nextPageId; initialize(); } - public void initialize() + private void initialize() { currentPageId = UNBOUND_PAGE_ID; current = null; @@ -70,14 +80,8 @@ public int getCurrentPageSize() @Override public boolean next() throws IOException { - if ( currentPageId == UNBOUND_PAGE_ID ) - { - currentPageId = 0; - } - else - { - currentPageId++; - } + currentPageId = nextPageId; + nextPageId++; assertPages(); byte[] page = page( currentPageId ); @@ -279,19 +283,29 @@ public void rewind() @Override public void close() { + if ( linkedCursor != null ) + { + linkedCursor.close(); + } current.close(); } @Override public boolean shouldRetry() throws IOException { - return current.shouldRetry(); + return linkedCursor != null && linkedCursor.shouldRetry() || current.shouldRetry(); } @Override public boolean checkAndClearBoundsFlag() { - return current.checkAndClearBoundsFlag(); + boolean result = false; + if ( linkedCursor != null ) + { + result = linkedCursor.checkAndClearBoundsFlag(); + } + result |= current.checkAndClearBoundsFlag(); + return result; } @Override @@ -321,7 +335,13 @@ public void clearCursorException() @Override public PageCursor openLinkedCursor( long pageId ) { - return current.openLinkedCursor( pageId ); + PageCursor toReturn = new PageAwareByteArrayCursor( pages, pageSize, pageId ); + if ( linkedCursor != null ) + { + linkedCursor.close(); + } + linkedCursor = toReturn; + return toReturn; } @Override diff --git a/community/index/src/test/java/org/neo4j/index/gbptree/PointerCheckingTest.java b/community/index/src/test/java/org/neo4j/index/gbptree/PointerCheckingTest.java index 29f1562d141b7..2142078702cd1 100644 --- a/community/index/src/test/java/org/neo4j/index/gbptree/PointerCheckingTest.java +++ b/community/index/src/test/java/org/neo4j/index/gbptree/PointerCheckingTest.java @@ -30,8 +30,13 @@ public class PointerCheckingTest { + private final PageCursor cursor = ByteArrayPageCursor.wrap( GenSafePointerPair.SIZE ); + private final int firstGeneration = 1; + private final int secondGeneration = 2; + private final int thirdGeneration = 3; + @Test - public void shouldThrowOnNoNode() throws Exception + public void checkChildShouldThrowOnNoNode() throws Exception { // WHEN try @@ -46,10 +51,9 @@ public void shouldThrowOnNoNode() throws Exception } @Test - public void shouldThrowOnReadFailure() throws Exception + public void checkChildShouldThrowOnReadFailure() throws Exception { // GIVEN - PageCursor cursor = ByteArrayPageCursor.wrap( GenSafePointerPair.SIZE ); long result = GenSafePointerPair.read( cursor, 0, 1 ); // WHEN @@ -65,17 +69,13 @@ public void shouldThrowOnReadFailure() throws Exception } @Test - public void shouldThrowOnWriteFailure() throws Exception + public void checkChildShouldThrowOnWriteFailure() throws Exception { // GIVEN - PageCursor cursor = ByteArrayPageCursor.wrap( GenSafePointerPair.SIZE ); - int firstGeneration = 1; - int secondGeneration = 2; - int thirdGeneration = 3; write( cursor, 123, 0, firstGeneration ); - cursor.setOffset( 0 ); + cursor.rewind(); write( cursor, 456, firstGeneration, secondGeneration ); - cursor.setOffset( 0 ); + cursor.rewind(); // WHEN // This write will see first and second written pointers and think they belong to CRASHed generation @@ -92,32 +92,95 @@ public void shouldThrowOnWriteFailure() throws Exception } @Test - public void shouldPassOnReadSuccess() throws Exception + public void checkChildShouldPassOnReadSuccess() throws Exception { // GIVEN - PageCursor cursor = ByteArrayPageCursor.wrap( GenSafePointerPair.SIZE ); - long generation = 1; - PointerChecking.checkChildPointer( write( cursor, 123, 0, generation ) ); - cursor.setOffset( 0 ); + PointerChecking.checkChildPointer( write( cursor, 123, 0, firstGeneration ) ); + cursor.rewind(); // WHEN - long result = read( cursor, 0, generation ); + long result = read( cursor, 0, firstGeneration ); // THEN PointerChecking.checkChildPointer( result ); } @Test - public void shouldPassOnWriteSuccess() throws Exception + public void checkChildShouldPassOnWriteSuccess() throws Exception + { + // WHEN + long result = write( cursor, 123, 0, firstGeneration ); + + // THEN + PointerChecking.checkChildPointer( result ); + } + + @Test + public void checkSiblingShouldPassOnReadSuccessForNoNodePointer() throws Exception { // GIVEN - PageCursor cursor = ByteArrayPageCursor.wrap( GenSafePointerPair.SIZE ); - long generation = 1; + write( cursor, TreeNode.NO_NODE_FLAG, firstGeneration, secondGeneration ); + cursor.rewind(); // WHEN - long result = write( cursor, 123, 0, generation ); + long result = read( cursor, firstGeneration, secondGeneration ); // THEN - PointerChecking.checkChildPointer( result ); + PointerChecking.checkSiblingPointer( result ); + } + + @Test + public void checkSiblingShouldPassOnReadSuccessForNodePointer() throws Exception + { + // GIVEN + long pointer = 101; + write( cursor, pointer, firstGeneration, secondGeneration ); + cursor.rewind(); + + // WHEN + long result = read( cursor, firstGeneration, secondGeneration ); + + // THEN + PointerChecking.checkSiblingPointer( result ); + } + + @Test + public void checkSiblingShouldThrowOnReadFailure() throws Exception + { + // WHEN + long result = read( cursor, firstGeneration, secondGeneration ); + + // WHEN + try + { + PointerChecking.checkSiblingPointer( result ); + fail( "Should have failed" ); + } + catch ( IllegalStateException e ) + { + // THEN good + } + } + + @Test + public void checkSiblingShouldThrowOnReadIllegalPointer() throws Exception + { + // GIVEN + GenSafePointer.write( cursor, secondGeneration, IdSpace.STATE_PAGE_A ); + cursor.rewind(); + + // WHEN + long result = read( cursor, firstGeneration, secondGeneration ); + + // WHEN + try + { + PointerChecking.checkSiblingPointer( result ); + fail( "Should have failed" ); + } + catch ( IllegalStateException e ) + { + // THEN good + } } } diff --git a/community/index/src/test/java/org/neo4j/index/gbptree/SeekCursorTest.java b/community/index/src/test/java/org/neo4j/index/gbptree/SeekCursorTest.java index 69f2dc8c80958..870ebc2fa148d 100644 --- a/community/index/src/test/java/org/neo4j/index/gbptree/SeekCursorTest.java +++ b/community/index/src/test/java/org/neo4j/index/gbptree/SeekCursorTest.java @@ -55,7 +55,6 @@ public class SeekCursorTest @Before public void setUp() throws IOException { - delegate.initialize(); pageCursor.next( 0L ); treeNode.initializeLeaf( pageCursor, STABLE_GENERATION, UNSTABLE_GENERATION ); }