Skip to content

Commit

Permalink
TreeNodeDynamicSize impl part 6 - internalOverflow
Browse files Browse the repository at this point in the history 
Also fix
- Some errors in tests
- Error in doSplitLeaf where leaf where not defraged properly
- Fix error where dead space was not updated correctly
  • Loading branch information
@burqen
burqen committed Jan 16, 2018
1 parent 859b1b6 commit e473322
Show file tree
Hide file tree
Showing 7 changed files with 112 additions and 71 deletions.
2 changes: 1 addition & 1 deletion community/index/src/main/java/org/neo4j/index/internal/gbptree/InternalTreeLogic.java
View file
Expand Up @@ -419,7 +419,7 @@ private void insertInInternal( PageCursor cursor, StructurePropagation<KEY> stru
createSuccessorIfNeeded( cursor, structurePropagation, UPDATE_MID_CHILD, createSuccessorIfNeeded( cursor, structurePropagation, UPDATE_MID_CHILD,
stableGeneration, unstableGeneration ); stableGeneration, unstableGeneration );


if ( bTreeNode.internalOverflow( keyCount ) ) if ( bTreeNode.internalOverflow( cursor, keyCount, primKey ) )
{ {
// Overflow // Overflow
// We will overwrite rightKey in structurePropagation, so copy it over to a place holder // We will overwrite rightKey in structurePropagation, so copy it over to a place holder
Expand Down
4 changes: 2 additions & 2 deletions community/index/src/main/java/org/neo4j/index/internal/gbptree/TreeNode.java
View file
Expand Up @@ -318,10 +318,10 @@ static void goTo( PageCursor cursor, String messageOnError, long nodeId )
/* SPLIT, MERGE AND REBALANCE */ /* SPLIT, MERGE AND REBALANCE */


/** /**
* Will internal overflow if inserting one more key? * Will internal overflow if inserting new key?
* @return true if leaf will overflow, else false. * @return true if leaf will overflow, else false.
*/ */
abstract boolean internalOverflow( int currentKeyCount ); abstract boolean internalOverflow( PageCursor cursor, int currentKeyCount, KEY newKey );


/** /**
* Will leaf overflow if inserting new key and value? * Will leaf overflow if inserting new key and value?
Expand Down
42 changes: 27 additions & 15 deletions community/index/src/main/java/org/neo4j/index/internal/gbptree/TreeNodeDynamicSize.java
View file
Expand Up @@ -56,7 +56,7 @@ public class TreeNodeDynamicSize<KEY, VALUE> extends TreeNode<KEY,VALUE>
{ {
super( pageSize, layout ); super( pageSize, layout );


keyValueSizeCap = (pageSize - HEADER_LENGTH_DYNAMIC) / LEAST_NUMBER_OF_ENTRIES_PER_PAGE - BYTE_SIZE_TOTAL_OVERHEAD; keyValueSizeCap = totalSpace( pageSize ) / LEAST_NUMBER_OF_ENTRIES_PER_PAGE - BYTE_SIZE_TOTAL_OVERHEAD;


if ( keyValueSizeCap < MINIMUM_ENTRY_SIZE_CAP ) if ( keyValueSizeCap < MINIMUM_ENTRY_SIZE_CAP )
{ {
Expand Down Expand Up @@ -155,7 +155,7 @@ void removeKeyValueAt( PageCursor cursor, int pos, int keyCount )


// Update dead space // Update dead space
int deadSpace = getDeadSpace( cursor ); int deadSpace = getDeadSpace( cursor );
setDeadSpace( cursor, deadSpace + keySize + valueSize + bytesKeySize() ); setDeadSpace( cursor, deadSpace + keySize + valueSize + bytesKeySize() + bytesValueSize() );


// Remove from offset array // Remove from offset array
removeSlotAt( cursor, pos, keyCount, keyPosOffsetLeaf( 0 ), bytesKeyOffset() ); removeSlotAt( cursor, pos, keyCount, keyPosOffsetLeaf( 0 ), bytesKeyOffset() );
Expand Down Expand Up @@ -256,7 +256,7 @@ int leafMaxKeyCount()
@Override @Override
boolean reasonableKeyCount( int keyCount ) boolean reasonableKeyCount( int keyCount )
{ {
throw new UnsupportedOperationException( "Implement me" ); return keyCount >= 0 && keyCount <= totalSpace( pageSize ) / BYTE_SIZE_TOTAL_OVERHEAD;
} }


@Override @Override
Expand All @@ -273,17 +273,21 @@ int childOffset( int pos )
} }


@Override @Override
boolean internalOverflow( int currentKeyCount ) boolean internalOverflow( PageCursor cursor, int currentKeyCount, KEY newKey )
{ {
throw new UnsupportedOperationException( "Implement me" ); // How much space do we have?
int allocSpace = getAllocSpace( cursor, currentKeyCount, INTERNAL );
int neededSpace = totalSpaceOfKeyChild( newKey );

return neededSpace > allocSpace;
} }


@Override @Override
Overflow leafOverflow( PageCursor cursor, int currentKeyCount, KEY newKey, VALUE newValue ) Overflow leafOverflow( PageCursor cursor, int currentKeyCount, KEY newKey, VALUE newValue )
{ {
// How much space do we have? // How much space do we have?
int deadSpace = getDeadSpace( cursor ); int deadSpace = getDeadSpace( cursor );
int allocSpace = getAllocSpace( cursor, currentKeyCount ); int allocSpace = getAllocSpace( cursor, currentKeyCount, LEAF );


// How much space do we need? // How much space do we need?
int keySize = layout.keySize( newKey ); int keySize = layout.keySize( newKey );
Expand Down Expand Up @@ -403,7 +407,7 @@ void defragmentLeaf( PageCursor cursor )
boolean leafUnderflow( PageCursor cursor, int keyCount ) boolean leafUnderflow( PageCursor cursor, int keyCount )
{ {
int halfSpace = halfSpace(); int halfSpace = halfSpace();
int allocSpace = getAllocSpace( cursor, keyCount ); int allocSpace = getAllocSpace( cursor, keyCount, LEAF );
int deadSpace = getDeadSpace( cursor ); int deadSpace = getDeadSpace( cursor );
int availableSpace = allocSpace + deadSpace; int availableSpace = allocSpace + deadSpace;


Expand Down Expand Up @@ -447,7 +451,6 @@ void doSplitLeaf( PageCursor leftCursor, int leftKeyCount, PageCursor rightCurso
// insert _,_,_,X,_,_,_,_,_,_,_ // insert _,_,_,X,_,_,_,_,_,_,_
// middle ^ // middle ^
moveKeysAndValues( leftCursor, middlePos - 1, rightCursor, 0, rightKeyCount ); moveKeysAndValues( leftCursor, middlePos - 1, rightCursor, 0, rightKeyCount );

defragmentLeaf( leftCursor ); defragmentLeaf( leftCursor );
insertKeyValueAt( leftCursor, newKey, newValue, insertPos, middlePos - 1 ); insertKeyValueAt( leftCursor, newKey, newValue, insertPos, middlePos - 1 );
} }
Expand All @@ -463,18 +466,17 @@ void doSplitLeaf( PageCursor leftCursor, int leftKeyCount, PageCursor rightCurso
int newInsertPos = insertPos - middlePos; int newInsertPos = insertPos - middlePos;
int keysToMove = leftKeyCount - middlePos; int keysToMove = leftKeyCount - middlePos;
moveKeysAndValues( leftCursor, middlePos, rightCursor, 0, keysToMove ); moveKeysAndValues( leftCursor, middlePos, rightCursor, 0, keysToMove );

defragmentLeaf( leftCursor );
// Insert new key and value
insertKeyValueAt( rightCursor, newKey, newValue, newInsertPos, keysToMove ); insertKeyValueAt( rightCursor, newKey, newValue, newInsertPos, keysToMove );
} }
TreeNode.setKeyCount( leftCursor, middlePos ); TreeNode.setKeyCount( leftCursor, middlePos );
TreeNode.setKeyCount( rightCursor, rightKeyCount ); TreeNode.setKeyCount( rightCursor, rightKeyCount );
} }


private int getAllocSpace( PageCursor cursor, int keyCount ) private int getAllocSpace( PageCursor cursor, int keyCount, Type type )
{ {
int allocOffset = getAllocOffset( cursor ); int allocOffset = getAllocOffset( cursor );
int endOfOffsetArray = keyPosOffsetLeaf( keyCount ); int endOfOffsetArray = type == LEAF ? keyPosOffsetLeaf( keyCount ) : keyPosOffsetInternal( keyCount );
return allocOffset - endOfOffsetArray; return allocOffset - endOfOffsetArray;
} }


Expand Down Expand Up @@ -572,16 +574,26 @@ private int middle( PageCursor leftCursor, int insertPos, KEY newKey, VALUE newV
return middle; return middle;
} }


private int totalSpace( int pageSize )
{
return pageSize - HEADER_LENGTH_DYNAMIC;
}

private int halfSpace() private int halfSpace()
{ {
return (pageSize - HEADER_LENGTH_DYNAMIC) / 2; return totalSpace( pageSize ) / 2;
} }


private int totalSpaceOfKeyValue( KEY key, VALUE value ) private int totalSpaceOfKeyValue( KEY key, VALUE value )
{ {
return bytesKeyOffset() + bytesKeySize() + bytesValueSize() + layout.keySize( key ) + layout.valueSize( value ); return bytesKeyOffset() + bytesKeySize() + bytesValueSize() + layout.keySize( key ) + layout.valueSize( value );
} }


private int totalSpaceOfKeyChild( KEY key )
{
return bytesKeyOffset() + bytesKeySize() + childSize() + layout.keySize( key );
}

private int totalSpaceOfKeyValue( PageCursor cursor, int pos ) private int totalSpaceOfKeyValue( PageCursor cursor, int pos )
{ {
placeCursorAtActualKey( cursor, pos, LEAF ); placeCursorAtActualKey( cursor, pos, LEAF );
Expand All @@ -604,10 +616,10 @@ void moveKeyValuesFromLeftToRight( PageCursor leftCursor, int leftKeyCount, Page
throw new UnsupportedOperationException( "Implement me" ); throw new UnsupportedOperationException( "Implement me" );
} }


private void setAllocOffset( PageCursor cursor, int allocSpace ) private void setAllocOffset( PageCursor cursor, int allocOffset )
{ {
cursor.setOffset( BYTE_POS_ALLOCOFFSET ); cursor.setOffset( BYTE_POS_ALLOCOFFSET );
putKeyOffset( cursor, allocSpace ); putKeyOffset( cursor, allocOffset );
} }


int getAllocOffset( PageCursor cursor ) int getAllocOffset( PageCursor cursor )
Expand Down
2 changes: 1 addition & 1 deletion community/index/src/main/java/org/neo4j/index/internal/gbptree/TreeNodeFixedSize.java
View file
Expand Up @@ -256,7 +256,7 @@ private int valueSize()
/* SPLIT, MERGE and REBALANCE*/ /* SPLIT, MERGE and REBALANCE*/


@Override @Override
boolean internalOverflow( int currentKeyCount ) boolean internalOverflow( PageCursor cursor, int currentKeyCount, KEY newKey )
{ {
return currentKeyCount + 1 > internalMaxKeyCount(); return currentKeyCount + 1 > internalMaxKeyCount();
} }
Expand Down
21 changes: 9 additions & 12 deletions ...nity/index/src/test/java/org/neo4j/index/internal/gbptree/CrashGenerationCleanerTest.java
View file
Expand Up @@ -66,9 +66,6 @@ public class CrashGenerationCleanerTest
private final int stableGeneration = 10; private final int stableGeneration = 10;
private final int unstableGeneration = 12; private final int unstableGeneration = 12;
private final int crashGeneration = 11; private final int crashGeneration = 11;
private final int firstChildPos = 0;
private final int secondChildPos = 1;
private final int thirdChildPos = 2;
private final List<PageCorruption> possibleCorruptionsInInternal = Arrays.asList( private final List<PageCorruption> possibleCorruptionsInInternal = Arrays.asList(
crashed( leftSibling() ), crashed( leftSibling() ),
crashed( rightSibling() ), crashed( rightSibling() ),
Expand Down Expand Up @@ -233,7 +230,7 @@ private void initializeFile( PagedFile pagedFile, Page... pages ) throws IOExcep
for ( Page page : pages ) for ( Page page : pages )
{ {
cursor.next(); cursor.next();
page.write( cursor, corruptibleTreeNode, stableGeneration, unstableGeneration, crashGeneration ); page.write( cursor, corruptibleTreeNode, layout, stableGeneration, unstableGeneration, crashGeneration );
} }
} }
} }
Expand Down Expand Up @@ -317,10 +314,10 @@ private Page( PageType type, PageCorruption... pageCorruptions )
this.pageCorruptions = pageCorruptions; this.pageCorruptions = pageCorruptions;
} }


private void write( PageCursor cursor, CorruptibleTreeNode node, int stableGeneration, int unstableGeneration, private void write( PageCursor cursor, CorruptibleTreeNode node, Layout<MutableLong,MutableLong> layout, int stableGeneration, int unstableGeneration,
int crashGeneration ) throws IOException int crashGeneration ) throws IOException
{ {
type.write( cursor, node, oldStableGeneration, stableGeneration ); type.write( cursor, node, layout, oldStableGeneration, stableGeneration );
Arrays.stream( pageCorruptions ) Arrays.stream( pageCorruptions )
.forEach( pc -> pc.corrupt( cursor, node, stableGeneration, unstableGeneration, crashGeneration ) ); .forEach( pc -> pc.corrupt( cursor, node, stableGeneration, unstableGeneration, crashGeneration ) );
} }
Expand All @@ -331,22 +328,22 @@ enum PageType
LEAF LEAF
{ {
@Override @Override
void write( PageCursor cursor, CorruptibleTreeNode corruptibleTreeNode, int stableGeneration, void write( PageCursor cursor, CorruptibleTreeNode corruptibleTreeNode, Layout<MutableLong,MutableLong> layout,
int unstableGeneration ) int stableGeneration, int unstableGeneration )
{ {
corruptibleTreeNode.initializeLeaf( cursor, stableGeneration, unstableGeneration ); corruptibleTreeNode.initializeLeaf( cursor, stableGeneration, unstableGeneration );
} }
}, },
INTERNAL INTERNAL
{ {
@Override @Override
void write( PageCursor cursor, CorruptibleTreeNode corruptibleTreeNode, int stableGeneration, void write( PageCursor cursor, CorruptibleTreeNode corruptibleTreeNode, Layout<MutableLong,MutableLong> layout,
int unstableGeneration ) int stableGeneration, int unstableGeneration )
{ {
corruptibleTreeNode.initializeInternal( cursor, stableGeneration, unstableGeneration ); corruptibleTreeNode.initializeInternal( cursor, stableGeneration, unstableGeneration );
long base = IdSpace.MIN_TREE_NODE_ID; long base = IdSpace.MIN_TREE_NODE_ID;
int keyCount; int keyCount;
for ( keyCount = 0; !corruptibleTreeNode.internalOverflow( keyCount ); keyCount++ ) for ( keyCount = 0; !corruptibleTreeNode.internalOverflow( cursor, keyCount, layout.newKey() ); keyCount++ )
{ {
long child = base + keyCount; long child = base + keyCount;
corruptibleTreeNode.setChildAt( cursor, child, keyCount, stableGeneration, unstableGeneration ); corruptibleTreeNode.setChildAt( cursor, child, keyCount, stableGeneration, unstableGeneration );
Expand All @@ -356,7 +353,7 @@ void write( PageCursor cursor, CorruptibleTreeNode corruptibleTreeNode, int stab
}; };


abstract void write( PageCursor cursor, CorruptibleTreeNode corruptibleTreeNode, abstract void write( PageCursor cursor, CorruptibleTreeNode corruptibleTreeNode,
int stableGeneration, int unstableGeneration ); Layout<MutableLong,MutableLong> layout, int stableGeneration, int unstableGeneration );
} }


/* GSPPType */ /* GSPPType */
Expand Down
16 changes: 8 additions & 8 deletions ...unity/index/src/test/java/org/neo4j/index/internal/gbptree/InternalTreeLogicTestBase.java
View file
Expand Up @@ -39,6 +39,7 @@
import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotEquals; import static org.junit.Assert.assertNotEquals;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertNull; import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertThat; import static org.junit.Assert.assertThat;
import static org.junit.Assert.assertTrue; import static org.junit.Assert.assertTrue;
Expand Down Expand Up @@ -1178,17 +1179,17 @@ public void shouldCreateNewVersionWhenRemoveInStableLeaf() throws Exception
// add some more keys to middleChild to not have remove trigger a merge // add some more keys to middleChild to not have remove trigger a merge
goTo( readCursor, middleChild ); goTo( readCursor, middleChild );
KEY firstKeyInMiddleChild = keyAt( 0, LEAF ); KEY firstKeyInMiddleChild = keyAt( 0, LEAF );
VALUE firstValueInMiddleChild = valueAt( 0 );
long seed = getSeed( firstKeyInMiddleChild ); long seed = getSeed( firstKeyInMiddleChild );
insert( key( seed + 1 ), value( seed + 1 ) ); insert( key( seed + 1 ), value( seed + 1 ) );
insert( key( seed + 3 ), value( seed + 3 ) );
goTo( readCursor, rootId ); goTo( readCursor, rootId );


assertSiblings( leftChild, middleChild, rightChild ); assertSiblings( leftChild, middleChild, rightChild );


// WHEN // WHEN
generationManager.checkpoint(); generationManager.checkpoint();
long middleKeySeed = i / 2; // Should be located in middle leaf assertNotNull( remove( firstKeyInMiddleChild, dontCare ) );
KEY middleKey = key( middleKeySeed );
remove( middleKey, dontCare );


// THEN // THEN
// root have new middle child // root have new middle child
Expand All @@ -1202,11 +1203,11 @@ public void shouldCreateNewVersionWhenRemoveInStableLeaf() throws Exception
assertEquals( newMiddleChild, successor( readCursor, stableGeneration, unstableGeneration ) ); assertEquals( newMiddleChild, successor( readCursor, stableGeneration, unstableGeneration ) );


// old middle child has seen no change // old middle child has seen no change
assertKeyAssociatedWithValue( middleKey, value( middleKeySeed ) ); assertKeyAssociatedWithValue( firstKeyInMiddleChild, firstValueInMiddleChild );


// new middle child has seen change // new middle child has seen change
goTo( readCursor, newMiddleChild ); goTo( readCursor, newMiddleChild );
assertKeyNotFound( middleKey, LEAF ); assertKeyNotFound( firstKeyInMiddleChild, LEAF );


// sibling pointers updated // sibling pointers updated
assertSiblings( leftChild, newMiddleChild, rightChild ); assertSiblings( leftChild, newMiddleChild, rightChild );
Expand Down Expand Up @@ -1553,7 +1554,7 @@ private void assertKeyAssociatedWithValue( KEY key, VALUE expectedValue )
assertTrue( KeySearch.isHit( search ) ); assertTrue( KeySearch.isHit( search ) );
int keyPos = KeySearch.positionOf( search ); int keyPos = KeySearch.positionOf( search );
node.valueAt( readCursor, readValue, keyPos ); node.valueAt( readCursor, readValue, keyPos );
assertEquals( expectedValue, readValue ); assertEqualsValue( expectedValue, readValue );
} }


private void assertKeyNotFound( KEY key, TreeNode.Type type ) private void assertKeyNotFound( KEY key, TreeNode.Type type )
Expand Down Expand Up @@ -1673,8 +1674,7 @@ private void insert( KEY key, VALUE value, ValueMerger<KEY,VALUE> valueMerger )
{ {
structurePropagation.hasRightKeyInsert = false; structurePropagation.hasRightKeyInsert = false;
structurePropagation.hasMidChildUpdate = false; structurePropagation.hasMidChildUpdate = false;
treeLogic.insert( cursor, structurePropagation, key, value, valueMerger, stableGeneration, treeLogic.insert( cursor, structurePropagation, key, value, valueMerger, stableGeneration, unstableGeneration );
unstableGeneration );
handleAfterChange(); handleAfterChange();
} }


Expand Down

0 comments on commit e473322

Please sign in to comment.