Skip to content

Commit

Permalink
Broke out state pages A/B from meta page
Browse files Browse the repository at this point in the history 
so that there are now one static meta page and state pages A/B
which work a bit like GSPP in that one of them is selected, even
selected differently depending on read/write use case

Valid state with highest generation will be picked when opening tree.
  • Loading branch information
@burqen @tinwelint
burqen authored and tinwelint committed Dec 6, 2016
1 parent 09b9c04 commit e57d2e8
Show file tree
Hide file tree
Showing 6 changed files with 689 additions and 85 deletions.
214 changes: 130 additions & 84 deletions community/index/src/main/java/org/neo4j/index/gbptree/GBPTree.java
View file
Expand Up @@ -19,6 +19,8 @@
*/
package org.neo4j.index.gbptree;

import org.apache.commons.lang3.tuple.Pair;

import java.io.File;
import java.io.IOException;
import java.nio.file.NoSuchFileException;
Expand Down Expand Up @@ -125,14 +127,14 @@ public class GBPTree<KEY,VALUE> implements Index<KEY,VALUE>, IdProvider
* Stable generation, i.e. generation which has survived the last {@link #flush()}.
* Unsigned int.
*/
private volatile long stableGeneration = 0;
private volatile long stableGeneration = GenSafePointer.MIN_GENERATION;

/**
* Unstable generation, i.e. the current generation under evolution. This generation will be the
* {@link #stableGeneration} in the next {@link #flush()}.
* Unsigned int.
*/
private volatile long unstableGeneration = 1;
private volatile long unstableGeneration = stableGeneration + 1;

/**
* Opens an index {@code indexFile} in the {@code pageCache}, creating and initializing it if it doesn't exist.
Expand Down Expand Up @@ -182,49 +184,9 @@ private PagedFile openOrCreate( PageCache pageCache, File indexFile,

try
{
// Read header
long layoutIdentifier;
int majorVersion;
int minorVersion;
try ( PageCursor metaCursor = openMetaPageCursor( pagedFile ) )
{
do
{
pageSize = metaCursor.getInt();
rootId = metaCursor.getLong();
lastId = metaCursor.getLong();
layoutIdentifier = metaCursor.getLong();
majorVersion = metaCursor.getInt();
minorVersion = metaCursor.getInt();
layout.readMetaData( metaCursor );
}
while ( metaCursor.shouldRetry() );
checkOutOfBounds( metaCursor );
}
if ( layoutIdentifier != layout.identifier() )
{
throw new IllegalArgumentException( "Tried to open " + indexFile + " using different layout "
+ layout.identifier() + " than what it was created with " + layoutIdentifier );
}
if ( majorVersion != layout.majorVersion() || minorVersion != layout.minorVersion() )
{
throw new IllegalArgumentException( "Index is of another version than the layout " +
"it tries to be opened with. Index version is [" + majorVersion + "." + minorVersion + "]" +
", but tried to load the index with version [" +
layout.majorVersion() + "." + layout.minorVersion() + "]" );
}
// This index was created with another page size, re-open with that actual page size
if ( pageSize != pageCache.pageSize() )
{
if ( pageSize > pageCache.pageSize() )
{
throw new IllegalStateException( "Index was created with page size:" + pageSize
+ ", but page cache used to open it this time has a smaller page size:"
+ pageCache.pageSize() + " so cannot be opened" );
}
pagedFile.close();
pagedFile = pageCache.map( indexFile, pageSize );
}
readMeta( indexFile, layout, pagedFile );
pagedFile = mapWithCorrectPageSize( pageCache, indexFile, pagedFile );
readState( pagedFile );
return pagedFile;
}
catch ( Throwable t )
Expand All @@ -242,6 +204,7 @@ private PagedFile openOrCreate( PageCache pageCache, File indexFile,
}
catch ( NoSuchFileException e )
{
// First time. Initiate meta and state page
pageSize = pageSizeForCreation == 0 ? pageCache.pageSize() : pageSizeForCreation;
if ( pageSize > pageCache.pageSize() )
{
Expand All @@ -254,23 +217,46 @@ private PagedFile openOrCreate( PageCache pageCache, File indexFile,
PagedFile pagedFile = pageCache.map( indexFile, pageSize, StandardOpenOption.CREATE );

// Write header
try ( PageCursor metaCursor = openMetaPageCursor( pagedFile ) )
{
metaCursor.putInt( pageSize );
metaCursor.putLong( rootId );
metaCursor.putLong( lastId );
metaCursor.putLong( layout.identifier() );
metaCursor.putInt( layout.majorVersion() );
metaCursor.putInt( layout.minorVersion() );
layout.writeMetaData( metaCursor );
checkOutOfBounds( metaCursor );
}
writeMeta( layout, pagedFile );
writeState( pagedFile );
pagedFile.flushAndForce();
created = true;
return pagedFile;
}
}

private void readState( PagedFile pagedFile ) throws IOException
{
try ( PageCursor cursor = pagedFile.io( 0L /*ignored*/, PagedFile.PF_SHARED_WRITE_LOCK ) )
{
Pair<TreeState,TreeState> states = TreeStatePair.readStatePages(
cursor, IdSpace.STATE_PAGE_A, IdSpace.STATE_PAGE_B );

TreeState state = TreeStatePair.selectNewestValidState( states );
rootId = state.rootId();
lastId = state.lastId();
stableGeneration = state.stableGeneration();
unstableGeneration = state.unstableGeneration();
}
}

private void writeState( PagedFile pagedFile ) throws IOException
{
try ( PageCursor cursor = pagedFile.io( 0L /*ignored*/, PagedFile.PF_SHARED_WRITE_LOCK ) )
{
Pair<TreeState,TreeState> states = TreeStatePair.readStatePages(
cursor, IdSpace.STATE_PAGE_A, IdSpace.STATE_PAGE_B );
TreeState oldestState = TreeStatePair.selectOldestOrInvalid( states );
long pageToOverwrite = oldestState.pageId();
if ( !cursor.next( pageToOverwrite ) )
{
throw new IllegalStateException( "Could not go to state page " + pageToOverwrite );
}
cursor.setOffset( 0 );
TreeState.write( cursor, stableGeneration, unstableGeneration, rootId, lastId );
}
}

private PageCursor openMetaPageCursor( PagedFile pagedFile ) throws IOException
{
PageCursor metaCursor = pagedFile.io( IdSpace.META_PAGE_ID, PagedFile.PF_SHARED_WRITE_LOCK );
Expand All @@ -281,6 +267,70 @@ private PageCursor openMetaPageCursor( PagedFile pagedFile ) throws IOException
return metaCursor;
}

private void readMeta( File indexFile, Layout<KEY,VALUE> layout, PagedFile pagedFile )
throws IOException
{
// Read meta
long layoutIdentifier;
int majorVersion;
int minorVersion;
try ( PageCursor metaCursor = openMetaPageCursor( pagedFile ) )
{
do
{
pageSize = metaCursor.getInt();
layoutIdentifier = metaCursor.getLong();
majorVersion = metaCursor.getInt();
minorVersion = metaCursor.getInt();
layout.readMetaData( metaCursor );
}
while ( metaCursor.shouldRetry() );
checkOutOfBounds( metaCursor );
}
if ( layoutIdentifier != layout.identifier() )
{
throw new IllegalArgumentException( "Tried to open " + indexFile + " using different layout "
+ layout.identifier() + " than what it was created with " + layoutIdentifier );
}
if ( majorVersion != layout.majorVersion() || minorVersion != layout.minorVersion() )
{
throw new IllegalArgumentException( "Index is of another version than the layout " +
"it tries to be opened with. Index version is [" + majorVersion + "." + minorVersion + "]" +
", but tried to load the index with version [" +
layout.majorVersion() + "." + layout.minorVersion() + "]" );
}
}

private void writeMeta( Layout<KEY,VALUE> layout, PagedFile pagedFile ) throws IOException
{
try ( PageCursor metaCursor = openMetaPageCursor( pagedFile ) )
{
metaCursor.putInt( pageSize );
metaCursor.putLong( layout.identifier() );
metaCursor.putInt( layout.majorVersion() );
metaCursor.putInt( layout.minorVersion() );
layout.writeMetaData( metaCursor );
checkOutOfBounds( metaCursor );
}
}

private PagedFile mapWithCorrectPageSize( PageCache pageCache, File indexFile, PagedFile pagedFile ) throws IOException
{
// This index was created with another page size, re-open with that actual page size
if ( pageSize != pageCache.pageSize() )
{
if ( pageSize > pageCache.pageSize() )
{
throw new IllegalStateException( "Index was created with page size:" + pageSize
+ ", but page cache used to open it this time has a smaller page size:"
+ pageCache.pageSize() + " so cannot be opened" );
}
pagedFile.close();
return pageCache.map( indexFile, pageSize );
}
return pagedFile;
}

@Override
public RawCursor<Hit<KEY,VALUE>,IOException> seek( KEY fromInclusive, KEY toExclusive ) throws IOException
{
Expand Down Expand Up @@ -338,37 +388,12 @@ public long acquireNewId()
return lastId;
}

// Utility method
public void printTree() throws IOException
{
try ( PageCursor cursor = pagedFile.io( rootId, PagedFile.PF_SHARED_READ_LOCK ) )
{
cursor.next();
TreePrinter.printTree( cursor, bTreeNode, layout, stableGeneration, unstableGeneration, System.out );
}
}

// Utility method
boolean consistencyCheck() throws IOException
{
try ( PageCursor cursor = pagedFile.io( rootId, PagedFile.PF_SHARED_READ_LOCK ) )
{
cursor.next();
return new ConsistencyChecker<>( bTreeNode, layout, stableGeneration, unstableGeneration )
.check( cursor );
}
}

@Override
public void flush() throws IOException
{
try ( PageCursor cursor = openMetaPageCursor( pagedFile ) )
{
cursor.putLong( 4, rootId );
cursor.putLong( 12, lastId );
// generations should be incremented as part of flush, but this functionality doesn't exist yet.
checkOutOfBounds( cursor );
}
stableGeneration = unstableGeneration;
unstableGeneration++;
writeState( pagedFile );
}

@Override
Expand Down Expand Up @@ -408,6 +433,27 @@ private void checkOutOfBounds( PageCursor cursor )
}
}

// Utility method
public void printTree() throws IOException
{
try ( PageCursor cursor = pagedFile.io( rootId, PagedFile.PF_SHARED_READ_LOCK ) )
{
cursor.next();
TreePrinter.printTree( cursor, bTreeNode, layout, stableGeneration, unstableGeneration, System.out );
}
}

// Utility method
boolean consistencyCheck() throws IOException
{
try ( PageCursor cursor = pagedFile.io( rootId, PagedFile.PF_SHARED_READ_LOCK ) )
{
cursor.next();
return new ConsistencyChecker<>( bTreeNode, layout, stableGeneration, unstableGeneration )
.check( cursor );
}
}

private class SingleIndexWriter implements IndexWriter<KEY,VALUE>
{
private final InternalTreeLogic<KEY,VALUE> treeLogic;
Expand Down

0 comments on commit e57d2e8

Please sign in to comment.