Skip to content

Commit

Permalink
GBPTree writer reinitialize InternalTreeLogic with correct split ratio
Browse files Browse the repository at this point in the history 
Tested in GBPTreeSingleWriterTest

Additional changes:
- Introduce monitor methods for tree growth and tree shrink to be able to
  create tests of desired height in test.
- Refactored TreePrinter to GBPTreeStructure. It will take a visitor and
  lead it through the tree one level at the time. You could also imagine
  traversing the tree BFS or DFS.
  To print tree you can use PrintingGBPTreeVisitor.
  It might be possible to hook ConsistencyChecker into this visitor pattern
  but that is left for future.
  • Loading branch information
@burqen
burqen committed Feb 7, 2019
1 parent 38e3e77 commit e1f9311
Show file tree
Hide file tree
Showing 9 changed files with 487 additions and 114 deletions.
46 changes: 38 additions & 8 deletions community/index/src/main/java/org/neo4j/index/internal/gbptree/GBPTree.java
View file
Expand Up @@ -174,6 +174,16 @@ public void cleanupFailed( Throwable throwable )
public void startupState( boolean clean ) public void startupState( boolean clean )
{ // no-op { // no-op
} }

@Override
public void treeGrowth()
{ // no-op
}

@Override
public void treeShrink()
{ // no-op
}
} }


/** /**
Expand Down Expand Up @@ -223,6 +233,16 @@ public void startupState( boolean clean )
* @param clean true if tree was clean on startup. * @param clean true if tree was clean on startup.
*/ */
void startupState( boolean clean ); void startupState( boolean clean );

/**
* Report tree growth, meaning split in root.
*/
void treeGrowth();

/**
* Report tree shrink, when root becomes empty.
*/
void treeShrink();
} }


/** /**
Expand Down Expand Up @@ -483,7 +503,7 @@ public GBPTree( PageCache pageCache, File indexFile, Layout<KEY,VALUE> layout, i
} }
this.bTreeNode = format.create( pageSize, layout ); this.bTreeNode = format.create( pageSize, layout );
this.freeList = new FreeListIdProvider( pagedFile, pageSize, rootId, FreeListIdProvider.NO_MONITOR ); this.freeList = new FreeListIdProvider( pagedFile, pageSize, rootId, FreeListIdProvider.NO_MONITOR );
this.writer = new SingleWriter( new InternalTreeLogic<>( freeList, bTreeNode, layout ) ); this.writer = new SingleWriter( new InternalTreeLogic<>( freeList, bTreeNode, layout, monitor ) );


// Create or load state // Create or load state
if ( created ) if ( created )
Expand Down Expand Up @@ -1110,6 +1130,15 @@ private CleanupJob createCleanupJob( RecoveryCleanupWorkCollector recoveryCleanu
} }
} }


void visit( GBPTreeVisitor<KEY,VALUE> visitor ) throws IOException
{
try ( PageCursor cursor = openRootCursor( PagedFile.PF_SHARED_READ_LOCK ) )
{
new GBPTreeStructure<>( bTreeNode, layout, stableGeneration( generation ), unstableGeneration( generation ) )
.visitTree( cursor, writer.cursor, visitor );
}
}

@SuppressWarnings( "unused" ) @SuppressWarnings( "unused" )
public void printTree() throws IOException public void printTree() throws IOException
{ {
Expand All @@ -1129,19 +1158,17 @@ public void printTree() throws IOException
@SuppressWarnings( "SameParameterValue" ) @SuppressWarnings( "SameParameterValue" )
void printTree( boolean printValues, boolean printPosition, boolean printState, boolean printHeader ) throws IOException void printTree( boolean printValues, boolean printPosition, boolean printState, boolean printHeader ) throws IOException
{ {
try ( PageCursor cursor = openRootCursor( PagedFile.PF_SHARED_READ_LOCK ) ) PrintingGBPTreeVisitor<KEY,VALUE> printingVisitor = new PrintingGBPTreeVisitor<>( System.out, printValues, printPosition, printState, printHeader );
{ visit( printingVisitor );
new TreePrinter<>( bTreeNode, layout, stableGeneration( generation ), unstableGeneration( generation ) )
.printTree( cursor, writer.cursor, System.out, printValues, printPosition, printState, printHeader );
}
} }


// Utility method // Utility method
public void printState() throws IOException public void printState() throws IOException
{ {
try ( PageCursor cursor = openRootCursor( PagedFile.PF_SHARED_READ_LOCK ) ) try ( PageCursor cursor = openRootCursor( PagedFile.PF_SHARED_READ_LOCK ) )
{ {
TreePrinter.printTreeState( cursor, System.out ); PrintingGBPTreeVisitor<KEY,VALUE> printingVisitor = new PrintingGBPTreeVisitor<>( System.out, false, false, true, false );
GBPTreeStructure.visitTreeState( cursor, printingVisitor );
} }
} }


Expand Down Expand Up @@ -1218,6 +1245,7 @@ private class SingleWriter implements Writer<KEY,VALUE>
// therefore safe to locally cache these generation fields from the volatile generation in the tree // therefore safe to locally cache these generation fields from the volatile generation in the tree
private long stableGeneration; private long stableGeneration;
private long unstableGeneration; private long unstableGeneration;
private double ratioToKeepInLeftOnSplit;


SingleWriter( InternalTreeLogic<KEY,VALUE> treeLogic ) SingleWriter( InternalTreeLogic<KEY,VALUE> treeLogic )
{ {
Expand Down Expand Up @@ -1260,6 +1288,7 @@ void initialize( double ratioToKeepInLeftOnSplit ) throws IOException
cursor = openRootCursor( PagedFile.PF_SHARED_WRITE_LOCK ); cursor = openRootCursor( PagedFile.PF_SHARED_WRITE_LOCK );
stableGeneration = stableGeneration( generation ); stableGeneration = stableGeneration( generation );
unstableGeneration = unstableGeneration( generation ); unstableGeneration = unstableGeneration( generation );
this.ratioToKeepInLeftOnSplit = ratioToKeepInLeftOnSplit;
assert assertNoSuccessor( cursor, stableGeneration, unstableGeneration ); assert assertNoSuccessor( cursor, stableGeneration, unstableGeneration );
treeLogic.initialize( cursor, ratioToKeepInLeftOnSplit ); treeLogic.initialize( cursor, ratioToKeepInLeftOnSplit );
success = true; success = true;
Expand Down Expand Up @@ -1312,7 +1341,7 @@ private void setRoot( long rootPointer )
{ {
long rootId = GenerationSafePointerPair.pointer( rootPointer ); long rootId = GenerationSafePointerPair.pointer( rootPointer );
GBPTree.this.setRoot( rootId, unstableGeneration ); GBPTree.this.setRoot( rootId, unstableGeneration );
treeLogic.initialize( cursor ); treeLogic.initialize( cursor, ratioToKeepInLeftOnSplit );
} }


@Override @Override
Expand Down Expand Up @@ -1356,6 +1385,7 @@ private void handleStructureChanges() throws IOException
stableGeneration, unstableGeneration ); stableGeneration, unstableGeneration );
TreeNode.setKeyCount( cursor, 1 ); TreeNode.setKeyCount( cursor, 1 );
setRoot( newRootId ); setRoot( newRootId );
monitor.treeGrowth();
} }
else if ( structurePropagation.hasMidChildUpdate ) else if ( structurePropagation.hasMidChildUpdate )
{ {
Expand Down
137 changes: 39 additions & 98 deletions ...j/index/internal/gbptree/TreePrinter.java → ...ex/internal/gbptree/GBPTreeStructure.java
View file
Expand Up @@ -23,41 +23,31 @@


import java.io.File; import java.io.File;
import java.io.IOException; import java.io.IOException;
import java.io.PrintStream;
import java.nio.file.StandardOpenOption; import java.nio.file.StandardOpenOption;


import org.neo4j.io.fs.FileSystemAbstraction;
import org.neo4j.io.pagecache.PageCache; import org.neo4j.io.pagecache.PageCache;
import org.neo4j.io.pagecache.PageCursor; import org.neo4j.io.pagecache.PageCursor;
import org.neo4j.io.pagecache.PagedFile; import org.neo4j.io.pagecache.PagedFile;
import org.neo4j.io.pagecache.impl.SingleFilePageSwapperFactory;
import org.neo4j.io.pagecache.impl.muninn.MuninnPageCache;
import org.neo4j.io.pagecache.tracing.cursor.PageCursorTracerSupplier;
import org.neo4j.io.pagecache.tracing.cursor.context.EmptyVersionContextSupplier;
import org.neo4j.scheduler.JobScheduler;


import static java.lang.String.format;
import static org.neo4j.graphdb.config.Configuration.EMPTY;
import static org.neo4j.index.internal.gbptree.ConsistencyChecker.assertOnTreeNode; import static org.neo4j.index.internal.gbptree.ConsistencyChecker.assertOnTreeNode;
import static org.neo4j.index.internal.gbptree.GenerationSafePointerPair.pointer; import static org.neo4j.index.internal.gbptree.GenerationSafePointerPair.pointer;
import static org.neo4j.index.internal.gbptree.TreeNode.Type.INTERNAL; import static org.neo4j.index.internal.gbptree.TreeNode.Type.INTERNAL;
import static org.neo4j.index.internal.gbptree.TreeNode.Type.LEAF; import static org.neo4j.index.internal.gbptree.TreeNode.Type.LEAF;
import static org.neo4j.io.pagecache.tracing.PageCacheTracer.NULL;


/** /**
* Utility class for printing a {@link GBPTree}, either whole or sub-tree. * Utility class for printing a {@link GBPTree}, either whole or sub-tree.
* *
* @param <KEY> type of keys in the tree. * @param <KEY> type of keys in the tree.
* @param <VALUE> type of values in the tree. * @param <VALUE> type of values in the tree.
*/ */
public class TreePrinter<KEY, VALUE> public class GBPTreeStructure<KEY, VALUE>
{ {
private final TreeNode<KEY,VALUE> node; private final TreeNode<KEY,VALUE> node;
private final Layout<KEY,VALUE> layout; private final Layout<KEY,VALUE> layout;
private final long stableGeneration; private final long stableGeneration;
private final long unstableGeneration; private final long unstableGeneration;


TreePrinter( TreeNode<KEY,VALUE> node, Layout<KEY,VALUE> layout, long stableGeneration, long unstableGeneration ) GBPTreeStructure( TreeNode<KEY,VALUE> node, Layout<KEY,VALUE> layout, long stableGeneration, long unstableGeneration )
{ {
this.node = node; this.node = node;
this.layout = layout; this.layout = layout;
Expand All @@ -66,87 +56,64 @@ public class TreePrinter<KEY, VALUE>
} }


/** /**
* Prints the header, that is tree state and meta information, about the tree present in the given {@code file}. * Visit the header, that is tree state and meta information, about the tree present in the given {@code file}.
*
* @param fs {@link FileSystemAbstraction} where the {@code file} exists.
* @param jobScheduler {@link JobScheduler} job scheduler to run page cache related tasks
* @param file {@link File} containing the tree to print header for.
* @param out {@link PrintStream} to print at.
* @throws IOException on I/O error.
*/
public static void printHeader( FileSystemAbstraction fs, JobScheduler jobScheduler, File file, PrintStream out ) throws IOException
{
SingleFilePageSwapperFactory swapper = new SingleFilePageSwapperFactory();
swapper.open( fs, EMPTY );
PageCursorTracerSupplier cursorTracerSupplier = PageCursorTracerSupplier.NULL;
try ( PageCache pageCache = new MuninnPageCache( swapper, 100, NULL, cursorTracerSupplier, EmptyVersionContextSupplier.EMPTY, jobScheduler ) )
{
printHeader( pageCache, file, out );
}
}

/**
* Prints the header, that is tree state and meta information, about the tree present in the given {@code file}.
* *
* @param pageCache {@link PageCache} able to map tree contained in {@code file}. * @param pageCache {@link PageCache} able to map tree contained in {@code file}.
* @param file {@link File} containing the tree to print header for. * @param file {@link File} containing the tree to print header for.
* @param out {@link PrintStream} to print at. * @param visitor {@link GBPTreeVisitor} that shall visit header.
* @throws IOException on I/O error. * @throws IOException on I/O error.
*/ */
private static void printHeader( PageCache pageCache, File file, PrintStream out ) throws IOException public static void visitHeader( PageCache pageCache, File file, GBPTreeVisitor visitor ) throws IOException
{ {
try ( PagedFile pagedFile = pageCache.map( file, pageCache.pageSize(), StandardOpenOption.READ ) ) try ( PagedFile pagedFile = pageCache.map( file, pageCache.pageSize(), StandardOpenOption.READ ) )
{ {
try ( PageCursor cursor = pagedFile.io( IdSpace.STATE_PAGE_A, PagedFile.PF_SHARED_READ_LOCK ) ) try ( PageCursor cursor = pagedFile.io( IdSpace.STATE_PAGE_A, PagedFile.PF_SHARED_READ_LOCK ) )
{ {
// TODO add printing of meta information here when that abstraction has been merged. // TODO add printing of meta information here when that abstraction has been merged.
printTreeState( cursor, out ); visitTreeState( cursor, visitor );
} }
} }
} }


static void printTreeState( PageCursor cursor, PrintStream out ) throws IOException static void visitTreeState( PageCursor cursor, GBPTreeVisitor visitor ) throws IOException
{ {
Pair<TreeState,TreeState> statePair = TreeStatePair.readStatePages( Pair<TreeState,TreeState> statePair = TreeStatePair.readStatePages(
cursor, IdSpace.STATE_PAGE_A, IdSpace.STATE_PAGE_B ); cursor, IdSpace.STATE_PAGE_A, IdSpace.STATE_PAGE_B );
out.println( "StateA: " + statePair.getLeft() ); visitor.treeState( statePair );
out.println( "StateB: " + statePair.getRight() );
} }


/** /**
* Prints a {@link GBPTree} in human readable form, very useful for debugging. * Let the passed in {@code cursor} point to the root or sub-tree (internal node) of what to visit.
* Let the passed in {@code cursor} point to the root or sub-tree (internal node) of what to print.
* Will print sub-tree from that point. Leaves cursor at same page as when called. No guarantees on offset.
* *
* @param cursor {@link PageCursor} placed at root of tree or sub-tree. * @param cursor {@link PageCursor} placed at root of tree or sub-tree.
* @param writeCursor Currently active {@link PageCursor write cursor} in tree. * @param writeCursor Currently active {@link PageCursor write cursor} in tree.
* @param out target to print tree at. * @param visitor {@link GBPTreeVisitor} that should visit the tree.
* @param printPosition whether or not to include positional (slot number) information.
* @param printState whether or not to also print state pages
* @param printHeader whether or not to also print header (type, generation, keyCount) of every node
* @throws IOException on page cache access error. * @throws IOException on page cache access error.
*/ */
void printTree( PageCursor cursor, PageCursor writeCursor, PrintStream out, boolean printValues, boolean printPosition, void visitTree( PageCursor cursor, PageCursor writeCursor, GBPTreeVisitor<KEY,VALUE> visitor ) throws IOException
boolean printState, boolean printHeader ) throws IOException
{ {
if ( printState ) // TreeState
{ long currentPage = cursor.getCurrentPageId();
long currentPage = cursor.getCurrentPageId(); Pair<TreeState,TreeState> statePair = TreeStatePair.readStatePages(
printTreeState( cursor, out ); cursor, IdSpace.STATE_PAGE_A, IdSpace.STATE_PAGE_B );
TreeNode.goTo( cursor, "back to tree node from reading state", currentPage ); visitor.treeState( statePair );
} TreeNode.goTo( cursor, "back to tree node from reading state", currentPage );

assertOnTreeNode( select( cursor, writeCursor ) ); assertOnTreeNode( select( cursor, writeCursor ) );


// Traverse the tree // Traverse the tree
int level = 0; int level = 0;
do do
{ {
// One level at the time // One level at the time
out.println( "Level " + level++ ); visitor.beginLevel( level );
long leftmostSibling = cursor.getCurrentPageId(); long leftmostSibling = cursor.getCurrentPageId();


// Go right through all siblings // Go right through all siblings
printLevel( cursor, writeCursor, out, printValues, printPosition, printHeader ); visitLevel( cursor, writeCursor, visitor );

visitor.endLevel( level );
level++;


// Then go back to the left-most node on this level // Then go back to the left-most node on this level
TreeNode.goTo( cursor, "back", leftmostSibling ); TreeNode.goTo( cursor, "back", leftmostSibling );
Expand All @@ -155,11 +122,12 @@ void printTree( PageCursor cursor, PageCursor writeCursor, PrintStream out, bool
while ( goToLeftmostChild( cursor, writeCursor ) ); while ( goToLeftmostChild( cursor, writeCursor ) );
} }


void printTreeNode( PageCursor cursor, PrintStream out, boolean printValues, boolean printPosition, private void visitTreeNode( PageCursor cursor, GBPTreeVisitor<KEY,VALUE> visitor ) throws IOException
boolean printHeader ) throws IOException
{ {
//[TYPE][GEN][KEYCOUNT] ([RIGHTSIBLING][LEFTSIBLING][SUCCESSOR]))
boolean isLeaf; boolean isLeaf;
int keyCount; int keyCount;
long generation = -1;
do do
{ {
isLeaf = TreeNode.isLeaf( cursor ); isLeaf = TreeNode.isLeaf( cursor );
Expand All @@ -168,27 +136,11 @@ void printTreeNode( PageCursor cursor, PrintStream out, boolean printValues, boo
{ {
cursor.setCursorException( "Unexpected keyCount " + keyCount ); cursor.setCursorException( "Unexpected keyCount " + keyCount );
} }
generation = TreeNode.generation( cursor );
} }
while ( cursor.shouldRetry() ); while ( cursor.shouldRetry() );
visitor.beginNode( cursor.getCurrentPageId(), isLeaf, generation, keyCount );


if ( printHeader )
{
//[TYPE][GEN][KEYCOUNT] ([RIGHTSIBLING][LEFTSIBLING][SUCCESSOR]))
long generation = -1;
do
{
generation = TreeNode.generation( cursor );

}
while ( cursor.shouldRetry() );
String treeNodeType = isLeaf ? "leaf" : "internal";
out.print( format( "{%d,%s,generation=%d,keyCount=%d}",
cursor.getCurrentPageId(), treeNodeType, generation, keyCount ) );
}
else
{
out.print( "{" + cursor.getCurrentPageId() + "} " );
}
KEY key = layout.newKey(); KEY key = layout.newKey();
VALUE value = layout.newValue(); VALUE value = layout.newValue();
for ( int i = 0; i < keyCount; i++ ) for ( int i = 0; i < keyCount; i++ )
Expand All @@ -208,24 +160,17 @@ void printTreeNode( PageCursor cursor, PrintStream out, boolean printValues, boo
} }
while ( cursor.shouldRetry() ); while ( cursor.shouldRetry() );


if ( printPosition ) visitor.position( i );
{
out.print( "#" + i + " " );
}

if ( isLeaf ) if ( isLeaf )
{ {
out.print( key ); visitor.key( key, isLeaf );
if ( printValues ) visitor.value( value );
{
out.print( "=" + value );
}
} }
else else
{ {
out.print( "/" + child + "\\ [" + key + "]" ); visitor.child( child );
visitor.key( key, isLeaf );
} }
out.print( " " );
} }
if ( !isLeaf ) if ( !isLeaf )
{ {
Expand All @@ -235,14 +180,10 @@ void printTreeNode( PageCursor cursor, PrintStream out, boolean printValues, boo
child = pointer( node.childAt( cursor, keyCount, stableGeneration, unstableGeneration ) ); child = pointer( node.childAt( cursor, keyCount, stableGeneration, unstableGeneration ) );
} }
while ( cursor.shouldRetry() ); while ( cursor.shouldRetry() );

visitor.position( keyCount );
if ( printPosition ) visitor.child( child );
{
out.print( "#" + keyCount + " " );
}
out.print( "/" + child + "\\" );
} }
out.println(); visitor.endNode( cursor.getCurrentPageId() );
} }


private boolean goToLeftmostChild( PageCursor readCursor, PageCursor writeCursor ) throws IOException private boolean goToLeftmostChild( PageCursor readCursor, PageCursor writeCursor ) throws IOException
Expand All @@ -267,14 +208,13 @@ private boolean goToLeftmostChild( PageCursor readCursor, PageCursor writeCursor
return isInternal; return isInternal;
} }


private void printLevel( PageCursor readCursor, PageCursor writeCursor, PrintStream out, boolean printValues, boolean printPosition, private void visitLevel( PageCursor readCursor, PageCursor writeCursor, GBPTreeVisitor<KEY,VALUE> visitor ) throws IOException
boolean printHeader ) throws IOException
{ {
long rightSibling = -1; long rightSibling = -1;
do do
{ {
PageCursor cursor = select( readCursor, writeCursor ); PageCursor cursor = select( readCursor, writeCursor );
printTreeNode( cursor, out, printValues, printPosition, printHeader ); visitTreeNode( cursor, visitor );


do do
{ {
Expand All @@ -295,4 +235,5 @@ private static PageCursor select( PageCursor readCursor, PageCursor writeCursor
return writeCursor == null ? readCursor : return writeCursor == null ? readCursor :
readCursor.getCurrentPageId() == writeCursor.getCurrentPageId() ? writeCursor : readCursor; readCursor.getCurrentPageId() == writeCursor.getCurrentPageId() ? writeCursor : readCursor;
} }

} }

0 comments on commit e1f9311

Please sign in to comment.