Skip to content

Commit

Permalink
GBPTree writer reinitialize InternalTreeLogic with correct split ratio
Browse files Browse the repository at this point in the history 
Tested in GBPTreeSingleWriterTest

Additional changes:
- Introduce monitor methods for tree growth and tree shrink to be able to
  create tests of desired height in test.
- Refactored TreePrinter to GBPTreeStructure. It will take a visitor and
  lead it through the tree one level at the time. You could also imagine
  traversing the tree BFS or DFS.
  To print tree you can use PrintingGBPTreeVisitor.
  It might be possible to hook ConsistencyChecker into this visitor pattern
  but that is left for future.
  • Loading branch information
@burqen
burqen committed Feb 7, 2019
1 parent 38e3e77 commit e1f9311
Show file tree
Hide file tree
Showing 9 changed files with 487 additions and 114 deletions.
46 changes: 38 additions & 8 deletions community/index/src/main/java/org/neo4j/index/internal/gbptree/GBPTree.java
View file
Expand Up @@ -174,6 +174,16 @@ public void cleanupFailed( Throwable throwable )
public void startupState( boolean clean )
{ // no-op
}

@Override
public void treeGrowth()
{ // no-op
}

@Override
public void treeShrink()
{ // no-op
}
}

/**
Expand Down Expand Up @@ -223,6 +233,16 @@ public void startupState( boolean clean )
* @param clean true if tree was clean on startup.
*/
void startupState( boolean clean );

/**
* Report tree growth, meaning split in root.
*/
void treeGrowth();

/**
* Report tree shrink, when root becomes empty.
*/
void treeShrink();
}

/**
Expand Down Expand Up @@ -483,7 +503,7 @@ public GBPTree( PageCache pageCache, File indexFile, Layout<KEY,VALUE> layout, i
}
this.bTreeNode = format.create( pageSize, layout );
this.freeList = new FreeListIdProvider( pagedFile, pageSize, rootId, FreeListIdProvider.NO_MONITOR );
this.writer = new SingleWriter( new InternalTreeLogic<>( freeList, bTreeNode, layout ) );
this.writer = new SingleWriter( new InternalTreeLogic<>( freeList, bTreeNode, layout, monitor ) );

// Create or load state
if ( created )
Expand Down Expand Up @@ -1110,6 +1130,15 @@ private CleanupJob createCleanupJob( RecoveryCleanupWorkCollector recoveryCleanu
}
}

void visit( GBPTreeVisitor<KEY,VALUE> visitor ) throws IOException
{
try ( PageCursor cursor = openRootCursor( PagedFile.PF_SHARED_READ_LOCK ) )
{
new GBPTreeStructure<>( bTreeNode, layout, stableGeneration( generation ), unstableGeneration( generation ) )
.visitTree( cursor, writer.cursor, visitor );
}
}

@SuppressWarnings( "unused" )
public void printTree() throws IOException
{
Expand All @@ -1129,19 +1158,17 @@ public void printTree() throws IOException
@SuppressWarnings( "SameParameterValue" )
void printTree( boolean printValues, boolean printPosition, boolean printState, boolean printHeader ) throws IOException
{
try ( PageCursor cursor = openRootCursor( PagedFile.PF_SHARED_READ_LOCK ) )
{
new TreePrinter<>( bTreeNode, layout, stableGeneration( generation ), unstableGeneration( generation ) )
.printTree( cursor, writer.cursor, System.out, printValues, printPosition, printState, printHeader );
}
PrintingGBPTreeVisitor<KEY,VALUE> printingVisitor = new PrintingGBPTreeVisitor<>( System.out, printValues, printPosition, printState, printHeader );
visit( printingVisitor );
}

// Utility method
public void printState() throws IOException
{
try ( PageCursor cursor = openRootCursor( PagedFile.PF_SHARED_READ_LOCK ) )
{
TreePrinter.printTreeState( cursor, System.out );
PrintingGBPTreeVisitor<KEY,VALUE> printingVisitor = new PrintingGBPTreeVisitor<>( System.out, false, false, true, false );
GBPTreeStructure.visitTreeState( cursor, printingVisitor );
}
}

Expand Down Expand Up @@ -1218,6 +1245,7 @@ private class SingleWriter implements Writer<KEY,VALUE>
// therefore safe to locally cache these generation fields from the volatile generation in the tree
private long stableGeneration;
private long unstableGeneration;
private double ratioToKeepInLeftOnSplit;

SingleWriter( InternalTreeLogic<KEY,VALUE> treeLogic )
{
Expand Down Expand Up @@ -1260,6 +1288,7 @@ void initialize( double ratioToKeepInLeftOnSplit ) throws IOException
cursor = openRootCursor( PagedFile.PF_SHARED_WRITE_LOCK );
stableGeneration = stableGeneration( generation );
unstableGeneration = unstableGeneration( generation );
this.ratioToKeepInLeftOnSplit = ratioToKeepInLeftOnSplit;
assert assertNoSuccessor( cursor, stableGeneration, unstableGeneration );
treeLogic.initialize( cursor, ratioToKeepInLeftOnSplit );
success = true;
Expand Down Expand Up @@ -1312,7 +1341,7 @@ private void setRoot( long rootPointer )
{
long rootId = GenerationSafePointerPair.pointer( rootPointer );
GBPTree.this.setRoot( rootId, unstableGeneration );
treeLogic.initialize( cursor );
treeLogic.initialize( cursor, ratioToKeepInLeftOnSplit );
}

@Override
Expand Down Expand Up @@ -1356,6 +1385,7 @@ private void handleStructureChanges() throws IOException
stableGeneration, unstableGeneration );
TreeNode.setKeyCount( cursor, 1 );
setRoot( newRootId );
monitor.treeGrowth();
}
else if ( structurePropagation.hasMidChildUpdate )
{
Expand Down
137 changes: 39 additions & 98 deletions ...j/index/internal/gbptree/TreePrinter.java → ...ex/internal/gbptree/GBPTreeStructure.java
View file
Expand Up @@ -23,41 +23,31 @@

import java.io.File;
import java.io.IOException;
import java.io.PrintStream;
import java.nio.file.StandardOpenOption;

import org.neo4j.io.fs.FileSystemAbstraction;
import org.neo4j.io.pagecache.PageCache;
import org.neo4j.io.pagecache.PageCursor;
import org.neo4j.io.pagecache.PagedFile;
import org.neo4j.io.pagecache.impl.SingleFilePageSwapperFactory;
import org.neo4j.io.pagecache.impl.muninn.MuninnPageCache;
import org.neo4j.io.pagecache.tracing.cursor.PageCursorTracerSupplier;
import org.neo4j.io.pagecache.tracing.cursor.context.EmptyVersionContextSupplier;
import org.neo4j.scheduler.JobScheduler;

import static java.lang.String.format;
import static org.neo4j.graphdb.config.Configuration.EMPTY;
import static org.neo4j.index.internal.gbptree.ConsistencyChecker.assertOnTreeNode;
import static org.neo4j.index.internal.gbptree.GenerationSafePointerPair.pointer;
import static org.neo4j.index.internal.gbptree.TreeNode.Type.INTERNAL;
import static org.neo4j.index.internal.gbptree.TreeNode.Type.LEAF;
import static org.neo4j.io.pagecache.tracing.PageCacheTracer.NULL;

/**
* Utility class for printing a {@link GBPTree}, either whole or sub-tree.
*
* @param <KEY> type of keys in the tree.
* @param <VALUE> type of values in the tree.
*/
public class TreePrinter<KEY, VALUE>
public class GBPTreeStructure<KEY, VALUE>
{
private final TreeNode<KEY,VALUE> node;
private final Layout<KEY,VALUE> layout;
private final long stableGeneration;
private final long unstableGeneration;

TreePrinter( TreeNode<KEY,VALUE> node, Layout<KEY,VALUE> layout, long stableGeneration, long unstableGeneration )
GBPTreeStructure( TreeNode<KEY,VALUE> node, Layout<KEY,VALUE> layout, long stableGeneration, long unstableGeneration )
{
this.node = node;
this.layout = layout;
Expand All @@ -66,87 +56,64 @@ public class TreePrinter<KEY, VALUE>
}

/**
* Prints the header, that is tree state and meta information, about the tree present in the given {@code file}.
*
* @param fs {@link FileSystemAbstraction} where the {@code file} exists.
* @param jobScheduler {@link JobScheduler} job scheduler to run page cache related tasks
* @param file {@link File} containing the tree to print header for.
* @param out {@link PrintStream} to print at.
* @throws IOException on I/O error.
*/
public static void printHeader( FileSystemAbstraction fs, JobScheduler jobScheduler, File file, PrintStream out ) throws IOException
{
SingleFilePageSwapperFactory swapper = new SingleFilePageSwapperFactory();
swapper.open( fs, EMPTY );
PageCursorTracerSupplier cursorTracerSupplier = PageCursorTracerSupplier.NULL;
try ( PageCache pageCache = new MuninnPageCache( swapper, 100, NULL, cursorTracerSupplier, EmptyVersionContextSupplier.EMPTY, jobScheduler ) )
{
printHeader( pageCache, file, out );
}
}

/**
* Prints the header, that is tree state and meta information, about the tree present in the given {@code file}.
* Visit the header, that is tree state and meta information, about the tree present in the given {@code file}.
*
* @param pageCache {@link PageCache} able to map tree contained in {@code file}.
* @param file {@link File} containing the tree to print header for.
* @param out {@link PrintStream} to print at.
* @param visitor {@link GBPTreeVisitor} that shall visit header.
* @throws IOException on I/O error.
*/
private static void printHeader( PageCache pageCache, File file, PrintStream out ) throws IOException
public static void visitHeader( PageCache pageCache, File file, GBPTreeVisitor visitor ) throws IOException
{
try ( PagedFile pagedFile = pageCache.map( file, pageCache.pageSize(), StandardOpenOption.READ ) )
{
try ( PageCursor cursor = pagedFile.io( IdSpace.STATE_PAGE_A, PagedFile.PF_SHARED_READ_LOCK ) )
{
// TODO add printing of meta information here when that abstraction has been merged.
printTreeState( cursor, out );
visitTreeState( cursor, visitor );
}
}
}

static void printTreeState( PageCursor cursor, PrintStream out ) throws IOException
static void visitTreeState( PageCursor cursor, GBPTreeVisitor visitor ) throws IOException
{
Pair<TreeState,TreeState> statePair = TreeStatePair.readStatePages(
cursor, IdSpace.STATE_PAGE_A, IdSpace.STATE_PAGE_B );
out.println( "StateA: " + statePair.getLeft() );
out.println( "StateB: " + statePair.getRight() );
visitor.treeState( statePair );
}

/**
* Prints a {@link GBPTree} in human readable form, very useful for debugging.
* Let the passed in {@code cursor} point to the root or sub-tree (internal node) of what to print.
* Will print sub-tree from that point. Leaves cursor at same page as when called. No guarantees on offset.
* Let the passed in {@code cursor} point to the root or sub-tree (internal node) of what to visit.
*
* @param cursor {@link PageCursor} placed at root of tree or sub-tree.
* @param writeCursor Currently active {@link PageCursor write cursor} in tree.
* @param out target to print tree at.
* @param printPosition whether or not to include positional (slot number) information.
* @param printState whether or not to also print state pages
* @param printHeader whether or not to also print header (type, generation, keyCount) of every node
* @param visitor {@link GBPTreeVisitor} that should visit the tree.
* @throws IOException on page cache access error.
*/
void printTree( PageCursor cursor, PageCursor writeCursor, PrintStream out, boolean printValues, boolean printPosition,
boolean printState, boolean printHeader ) throws IOException
void visitTree( PageCursor cursor, PageCursor writeCursor, GBPTreeVisitor<KEY,VALUE> visitor ) throws IOException
{
if ( printState )
{
long currentPage = cursor.getCurrentPageId();
printTreeState( cursor, out );
TreeNode.goTo( cursor, "back to tree node from reading state", currentPage );
}
// TreeState
long currentPage = cursor.getCurrentPageId();
Pair<TreeState,TreeState> statePair = TreeStatePair.readStatePages(
cursor, IdSpace.STATE_PAGE_A, IdSpace.STATE_PAGE_B );
visitor.treeState( statePair );
TreeNode.goTo( cursor, "back to tree node from reading state", currentPage );

assertOnTreeNode( select( cursor, writeCursor ) );

// Traverse the tree
int level = 0;
do
{
// One level at the time
out.println( "Level " + level++ );
visitor.beginLevel( level );
long leftmostSibling = cursor.getCurrentPageId();

// Go right through all siblings
printLevel( cursor, writeCursor, out, printValues, printPosition, printHeader );
visitLevel( cursor, writeCursor, visitor );

visitor.endLevel( level );
level++;

// Then go back to the left-most node on this level
TreeNode.goTo( cursor, "back", leftmostSibling );
Expand All @@ -155,11 +122,12 @@ void printTree( PageCursor cursor, PageCursor writeCursor, PrintStream out, bool
while ( goToLeftmostChild( cursor, writeCursor ) );
}

void printTreeNode( PageCursor cursor, PrintStream out, boolean printValues, boolean printPosition,
boolean printHeader ) throws IOException
private void visitTreeNode( PageCursor cursor, GBPTreeVisitor<KEY,VALUE> visitor ) throws IOException
{
//[TYPE][GEN][KEYCOUNT] ([RIGHTSIBLING][LEFTSIBLING][SUCCESSOR]))
boolean isLeaf;
int keyCount;
long generation = -1;
do
{
isLeaf = TreeNode.isLeaf( cursor );
Expand All @@ -168,27 +136,11 @@ void printTreeNode( PageCursor cursor, PrintStream out, boolean printValues, boo
{
cursor.setCursorException( "Unexpected keyCount " + keyCount );
}
generation = TreeNode.generation( cursor );
}
while ( cursor.shouldRetry() );
visitor.beginNode( cursor.getCurrentPageId(), isLeaf, generation, keyCount );

if ( printHeader )
{
//[TYPE][GEN][KEYCOUNT] ([RIGHTSIBLING][LEFTSIBLING][SUCCESSOR]))
long generation = -1;
do
{
generation = TreeNode.generation( cursor );

}
while ( cursor.shouldRetry() );
String treeNodeType = isLeaf ? "leaf" : "internal";
out.print( format( "{%d,%s,generation=%d,keyCount=%d}",
cursor.getCurrentPageId(), treeNodeType, generation, keyCount ) );
}
else
{
out.print( "{" + cursor.getCurrentPageId() + "} " );
}
KEY key = layout.newKey();
VALUE value = layout.newValue();
for ( int i = 0; i < keyCount; i++ )
Expand All @@ -208,24 +160,17 @@ void printTreeNode( PageCursor cursor, PrintStream out, boolean printValues, boo
}
while ( cursor.shouldRetry() );

if ( printPosition )
{
out.print( "#" + i + " " );
}

visitor.position( i );
if ( isLeaf )
{
out.print( key );
if ( printValues )
{
out.print( "=" + value );
}
visitor.key( key, isLeaf );
visitor.value( value );
}
else
{
out.print( "/" + child + "\\ [" + key + "]" );
visitor.child( child );
visitor.key( key, isLeaf );
}
out.print( " " );
}
if ( !isLeaf )
{
Expand All @@ -235,14 +180,10 @@ void printTreeNode( PageCursor cursor, PrintStream out, boolean printValues, boo
child = pointer( node.childAt( cursor, keyCount, stableGeneration, unstableGeneration ) );
}
while ( cursor.shouldRetry() );

if ( printPosition )
{
out.print( "#" + keyCount + " " );
}
out.print( "/" + child + "\\" );
visitor.position( keyCount );
visitor.child( child );
}
out.println();
visitor.endNode( cursor.getCurrentPageId() );
}

private boolean goToLeftmostChild( PageCursor readCursor, PageCursor writeCursor ) throws IOException
Expand All @@ -267,14 +208,13 @@ private boolean goToLeftmostChild( PageCursor readCursor, PageCursor writeCursor
return isInternal;
}

private void printLevel( PageCursor readCursor, PageCursor writeCursor, PrintStream out, boolean printValues, boolean printPosition,
boolean printHeader ) throws IOException
private void visitLevel( PageCursor readCursor, PageCursor writeCursor, GBPTreeVisitor<KEY,VALUE> visitor ) throws IOException
{
long rightSibling = -1;
do
{
PageCursor cursor = select( readCursor, writeCursor );
printTreeNode( cursor, out, printValues, printPosition, printHeader );
visitTreeNode( cursor, visitor );

do
{
Expand All @@ -295,4 +235,5 @@ private static PageCursor select( PageCursor readCursor, PageCursor writeCursor
return writeCursor == null ? readCursor :
readCursor.getCurrentPageId() == writeCursor.getCurrentPageId() ? writeCursor : readCursor;
}

}

0 comments on commit e1f9311

Please sign in to comment.