Skip to content

Commit

Permalink
More fine grained versioning of GBPTree format
Browse files Browse the repository at this point in the history 
Previously there was this global FORMAT_VERSION that controlled a single format version
of the whole GBPTree (and index component in general). Now with the introduction of
dynamic-size format in additional to the fixed-size this required a change because
there's now two active formats concurrently.

This change goes a bit further than that in that it splits up the version int
in four pieces, four different versions of a GBPTree. Two of them are used now,
which are format identifier and format version. Only the format identifier is used
now, sort of, but soon perhaps the format version will also come into play.
The two remaining versions are left as unused for now. Format identifier for
fixed-size format is kept at 2, which was the old FORMAT_VERSION and its
version set to 0. This keeps backwards compatibility with existing indexes.

Any existing dynamic-size indexes created before this commit will have to be recreated,
but in practice this will not be a problem because it shouldn't really be used anywhere.

Logic for open/create in GBPTree constructor has had some slight face lift to be
able to handle this new format diversity, introducing Meta class for reading/writing
and also verifying meta page.
  • Loading branch information
@tinwelint
tinwelint committed Jan 22, 2018
1 parent 0bc9846 commit e0e0115
Show file tree
Hide file tree
Showing 19 changed files with 538 additions and 148 deletions.
104 changes: 27 additions & 77 deletions community/index/src/main/java/org/neo4j/index/internal/gbptree/GBPTree.java
View file
Expand Up @@ -36,7 +36,6 @@
import org.neo4j.collection.primitive.PrimitiveLongSet;
import org.neo4j.cursor.RawCursor;
import org.neo4j.helpers.Exceptions;
import org.neo4j.io.pagecache.CursorException;
import org.neo4j.io.pagecache.IOLimiter;
import org.neo4j.io.pagecache.PageCache;
import org.neo4j.io.pagecache.PageCursor;
Expand Down Expand Up @@ -124,19 +123,6 @@
*/
public class GBPTree<KEY,VALUE> implements Closeable
{
/**
* Version of the format that makes up the tree. This includes:
* <ul>
* <li>{@link TreeNode} format, header, keys, children, values</li>
* <li>{@link GenerationSafePointer} and {@link GenerationSafePointerPair}</li>
* <li>{@link IdSpace} i.e. which pages are fixed</li>
* <li>{@link TreeState} and {@link TreeStatePair}</li>
* </ul>
* If any of the above changes the on-page format then this version should be bumped, so that opening
* an index on wrong format version fails and user will need to rebuild.
*/
static final int FORMAT_VERSION = 2;

/**
* For monitoring {@link GBPTree}.
*/
Expand Down Expand Up @@ -424,11 +410,22 @@ public GBPTree( PageCache pageCache, File indexFile, Layout<KEY,VALUE> layout, i
boolean success = false;
try
{
this.pagedFile = openOrCreate( pageCache, indexFile, tentativePageSize, layout );
this.pagedFile = openOrCreate( pageCache, indexFile, tentativePageSize );
this.pageSize = pagedFile.pageSize();
closed = false;
this.bTreeNode = layout.fixedSize() ? new TreeNodeFixedSize<>( pageSize, layout )
: new TreeNodeDynamicSize<>( pageSize, layout );
TreeNodeSelector.Factory format;
if ( created )
{
format = TreeNodeSelector.selectByLayout( layout );
writeMeta( layout, format, pagedFile );
}
else
{
Meta meta = readMeta( layout, pagedFile );
meta.verify( layout );
format = TreeNodeSelector.selectByFormat( meta.getFormatIdentifier(), meta.getFormatVersion() );
}
this.bTreeNode = format.create( pageSize, layout );
this.freeList = new FreeListIdProvider( pagedFile, pageSize, rootId, FreeListIdProvider.NO_MONITOR );
this.writer = new SingleWriter( new InternalTreeLogic<>( freeList, bTreeNode, layout ) );

Expand Down Expand Up @@ -468,9 +465,6 @@ public GBPTree( PageCache pageCache, File indexFile, Layout<KEY,VALUE> layout, i

private void initializeAfterCreation( Layout<KEY,VALUE> layout, Consumer<PageCursor> headerWriter ) throws IOException
{
// Initialize meta
writeMeta( layout, pagedFile );

// Initialize state
try ( PageCursor cursor = pagedFile.io( 0 /*ignored*/, PagedFile.PF_SHARED_WRITE_LOCK ) )
{
Expand All @@ -496,29 +490,29 @@ private void initializeAfterCreation( Layout<KEY,VALUE> layout, Consumer<PageCur
}

private PagedFile openOrCreate( PageCache pageCache, File indexFile,
int pageSizeForCreation, Layout<KEY,VALUE> layout ) throws IOException
int pageSizeForCreation ) throws IOException
{
try
{
return openExistingIndexFile( pageCache, indexFile, layout );
return openExistingIndexFile( pageCache, indexFile );
}
catch ( NoSuchFileException e )
{
return createNewIndexFile( pageCache, indexFile, pageSizeForCreation );
}
}

private static <KEY, VALUE> PagedFile openExistingIndexFile( PageCache pageCache, File indexFile, Layout<KEY,VALUE> layout )
throws IOException
private static PagedFile openExistingIndexFile( PageCache pageCache, File indexFile ) throws IOException
{
PagedFile pagedFile = pageCache.map( indexFile, pageCache.pageSize() );
// This index already exists, verify meta data aligns with expectations

boolean success = false;
try
{
int pageSize = readMeta( layout, pagedFile );
pagedFile = mapWithCorrectPageSize( pageCache, indexFile, pagedFile, pageSize );
// We're only interested in the page size really, so don't involve layout at this point
Meta meta = readMeta( null, pagedFile );
pagedFile = mapWithCorrectPageSize( pageCache, indexFile, pagedFile, meta.getPageSize() );
success = true;
return pagedFile;
}
Expand Down Expand Up @@ -588,8 +582,9 @@ private void loadState( PagedFile pagedFile, Header.Reader headerReader ) throws
*/
public static void readHeader( PageCache pageCache, File indexFile, Layout<?,?> layout, Header.Reader headerReader ) throws IOException
{
try ( PagedFile pagedFile = openExistingIndexFile( pageCache, indexFile, layout ) )
try ( PagedFile pagedFile = openExistingIndexFile( pageCache, indexFile ) )
{
readMeta( layout, pagedFile ).verify( layout );
Pair<TreeState,TreeState> states = loadStatePages( pagedFile );
TreeState state = TreeStatePair.selectNewestValidState( states );
try ( PageCursor cursor = pagedFile.io( state.pageId(), PagedFile.PF_SHARED_READ_LOCK ) )
Expand Down Expand Up @@ -731,66 +726,21 @@ private static PageCursor openMetaPageCursor( PagedFile pagedFile, int pfFlags )
return metaCursor;
}

private static <KEY,VALUE> int readMeta( Layout<KEY,VALUE> layout, PagedFile pagedFile )
private static <KEY,VALUE> Meta readMeta( Layout<KEY,VALUE> layout, PagedFile pagedFile )
throws IOException
{
// Read meta
int formatVersion;
int pageSize;
long layoutIdentifier;
int majorVersion;
int minorVersion;
try ( PageCursor metaCursor = openMetaPageCursor( pagedFile, PagedFile.PF_SHARED_READ_LOCK ) )
{
do
{
formatVersion = metaCursor.getInt();
pageSize = metaCursor.getInt();
layoutIdentifier = metaCursor.getLong();
majorVersion = metaCursor.getInt();
minorVersion = metaCursor.getInt();
layout.readMetaData( metaCursor );
}
while ( metaCursor.shouldRetry() );
checkOutOfBounds( metaCursor );
metaCursor.checkAndClearCursorException();
}
catch ( CursorException e )
{
throw new MetadataMismatchException( e,
"Tried to open, but caught an error while reading meta data. " +
"File is expected to be corrupt, try to rebuild." );
}

if ( formatVersion != FORMAT_VERSION )
{
throw new MetadataMismatchException(
"Tried to open with a different format version than " +
"what it was created with. Created with %d, opened with %d",
formatVersion, FORMAT_VERSION );
}
if ( !layout.compatibleWith( layoutIdentifier, majorVersion, minorVersion ) )
{
throw new MetadataMismatchException(
"Tried to open using layout not compatible with " +
"what the index was created with. Created with: layoutIdentifier=%d,majorVersion=%d,minorVersion=%d. " +
"Opened with layoutIdentifier=%d,majorVersion=%d,minorVersion=%d",
layoutIdentifier, majorVersion, minorVersion, layout.identifier(), layout.majorVersion(), layout.minorVersion() );
return Meta.read( metaCursor, layout );
}
return pageSize;
}

private void writeMeta( Layout<KEY,VALUE> layout, PagedFile pagedFile ) throws IOException
private void writeMeta( Layout<KEY,VALUE> layout, TreeNodeSelector.Factory format, PagedFile pagedFile ) throws IOException
{
Meta meta = new Meta( format.formatIdentifier(), format.formatVersion(), pageSize, layout );
try ( PageCursor metaCursor = openMetaPageCursor( pagedFile, PagedFile.PF_SHARED_WRITE_LOCK ) )
{
metaCursor.putInt( FORMAT_VERSION );
metaCursor.putInt( pageSize );
metaCursor.putLong( layout.identifier() );
metaCursor.putInt( layout.majorVersion() );
metaCursor.putInt( layout.minorVersion() );
layout.writeMetaData( metaCursor );
checkOutOfBounds( metaCursor );
meta.write( metaCursor, layout );
}
}

Expand Down

0 comments on commit e0e0115

Please sign in to comment.