Skip to content

Commit

Permalink
NativeSchemaIndexProvider: Step1
Browse files Browse the repository at this point in the history 
- getPopulator
- getOnlineAccessor
- getPopulationFailure

Note: new 'readHeader' functionality in GBPTree to be
able to read stored header without opening the tree
for writes.
  • Loading branch information
@burqen @tinwelint
burqen authored and tinwelint committed Aug 6, 2017
1 parent e9aeb33 commit 077d500
Show file tree
Hide file tree
Showing 11 changed files with 811 additions and 85 deletions.
126 changes: 80 additions & 46 deletions community/index/src/main/java/org/neo4j/index/internal/gbptree/GBPTree.java
View file
Expand Up @@ -391,6 +391,7 @@ public GBPTree( PageCache pageCache, File indexFile, Layout<KEY,VALUE> layout, i
try try
{ {
this.pagedFile = openOrCreate( pageCache, indexFile, tentativePageSize, layout ); this.pagedFile = openOrCreate( pageCache, indexFile, tentativePageSize, layout );
this.pageSize = pagedFile.pageSize();
closed = false; closed = false;
this.bTreeNode = new TreeNode<>( pageSize, layout ); this.bTreeNode = new TreeNode<>( pageSize, layout );
this.freeList = new FreeListIdProvider( pagedFile, pageSize, rootId, FreeListIdProvider.NO_MONITOR ); this.freeList = new FreeListIdProvider( pagedFile, pageSize, rootId, FreeListIdProvider.NO_MONITOR );
Expand Down Expand Up @@ -462,46 +463,57 @@ private PagedFile openOrCreate( PageCache pageCache, File indexFile,
{ {
try try
{ {
PagedFile pagedFile = pageCache.map( indexFile, pageCache.pageSize() ); return openExistingIndexFile( pageCache, indexFile, layout );
// This index already exists, verify the header with what we got passed into the constructor this time }
catch ( NoSuchFileException e )
{
return createNewIndexFile( pageCache, indexFile, pageSizeForCreation );
}
}


private static <KEY, VALUE> PagedFile openExistingIndexFile( PageCache pageCache, File indexFile, Layout<KEY,VALUE> layout )
throws IOException
{
PagedFile pagedFile = pageCache.map( indexFile, pageCache.pageSize() );
// This index already exists, verify meta data aligns with expectations

try
{
int pageSize = readMeta( layout, pagedFile );
pagedFile = mapWithCorrectPageSize( pageCache, indexFile, pagedFile, pageSize );
return pagedFile;
}
catch ( Throwable t )
{
try try
{ {
readMeta( layout, pagedFile ); pagedFile.close();
pagedFile = mapWithCorrectPageSize( pageCache, indexFile, pagedFile );
return pagedFile;
} }
catch ( Throwable t ) catch ( IOException e )
{ {
try t.addSuppressed( e );
{
pagedFile.close();
}
catch ( IOException e )
{
t.addSuppressed( e );
}
throw t;
} }
throw t;
} }
catch ( NoSuchFileException e ) }
{
// First time
monitor.noStoreFile();
pageSize = pageSizeForCreation == 0 ? pageCache.pageSize() : pageSizeForCreation;
if ( pageSize > pageCache.pageSize() )
{
throw new MetadataMismatchException(
"Tried to create tree with page size %d" +
", but page cache used to create it has a smaller page size %d" +
" so cannot be created", pageSize, pageCache.pageSize() );
}


// We need to create this index private PagedFile createNewIndexFile( PageCache pageCache, File indexFile, int pageSizeForCreation ) throws IOException
PagedFile pagedFile = pageCache.map( indexFile, pageSize, StandardOpenOption.CREATE ); {
created = true; // First time
return pagedFile; monitor.noStoreFile();
int pageSize = pageSizeForCreation == 0 ? pageCache.pageSize() : pageSizeForCreation;
if ( pageSize > pageCache.pageSize() )
{
throw new MetadataMismatchException(
"Tried to create tree with page size %d" +
", but page cache used to create it has a smaller page size %d" +
" so cannot be created", pageSize, pageCache.pageSize() );
} }

// We need to create this index
PagedFile pagedFile = pageCache.map( indexFile, pageSize, StandardOpenOption.CREATE );
created = true;
return pagedFile;
} }


private void loadState( PagedFile pagedFile, Header.Reader headerReader ) throws IOException private void loadState( PagedFile pagedFile, Header.Reader headerReader ) throws IOException
Expand All @@ -511,7 +523,7 @@ private void loadState( PagedFile pagedFile, Header.Reader headerReader ) throws
try ( PageCursor cursor = pagedFile.io( state.pageId(), PagedFile.PF_SHARED_READ_LOCK ) ) try ( PageCursor cursor = pagedFile.io( state.pageId(), PagedFile.PF_SHARED_READ_LOCK ) )
{ {
PageCursorUtil.goTo( cursor, "header data", state.pageId() ); PageCursorUtil.goTo( cursor, "header data", state.pageId() );
readHeader( headerReader, cursor ); doReadHeader( headerReader, cursor );
} }
generation = Generation.generation( state.stableGeneration(), state.unstableGeneration() ); generation = Generation.generation( state.stableGeneration(), state.unstableGeneration() );
setRoot( state.rootId(), state.rootGeneration() ); setRoot( state.rootId(), state.rootGeneration() );
Expand All @@ -525,7 +537,32 @@ private void loadState( PagedFile pagedFile, Header.Reader headerReader ) throws
clean = state.isClean(); clean = state.isClean();
} }


private static void readHeader( Header.Reader headerReader, PageCursor cursor ) throws IOException /**
* Use when you are only interested in reading the header of existing index file without opening the index for writes.
* Useful when reading header and the demands on matching layout can be relaxed a bit.
*
* @param pageCache {@link PageCache} to use to map index file
* @param indexFile {@link File} containing the actual index
* @param layout {@link Layout} only used to verify compatibility with stored identifier and version. Can be 'dummy' implementation.
* @param headerReader reads header data, previously written using {@link #checkpoint(IOLimiter, Consumer)}
* or {@link #close()}
* @throws IOException On page cache error
*/
public static void readHeader( PageCache pageCache, File indexFile, Layout<?,?> layout, Header.Reader headerReader ) throws IOException
{
try ( PagedFile pagedFile = openExistingIndexFile( pageCache, indexFile, layout ) )
{
Pair<TreeState,TreeState> states = readStatePages( pagedFile );
TreeState state = TreeStatePair.selectNewestValidState( states );
try ( PageCursor cursor = pagedFile.io( state.pageId(), PagedFile.PF_SHARED_READ_LOCK ) )
{
PageCursorUtil.goTo( cursor, "header data", state.pageId() );
doReadHeader( headerReader, cursor );
}
}
}

private static void doReadHeader( Header.Reader headerReader, PageCursor cursor ) throws IOException
{ {
int headerDataLength; int headerDataLength;
do do
Expand Down Expand Up @@ -623,10 +660,12 @@ private static PageCursor openMetaPageCursor( PagedFile pagedFile, int pfFlags )
return metaCursor; return metaCursor;
} }


private void readMeta( Layout<KEY,VALUE> layout, PagedFile pagedFile ) throws IOException private static <KEY,VALUE> int readMeta( Layout<KEY,VALUE> layout, PagedFile pagedFile )
throws IOException
{ {
// Read meta // Read meta
int formatVersion; int formatVersion;
int pageSize;
long layoutIdentifier; long layoutIdentifier;
int majorVersion; int majorVersion;
int minorVersion; int minorVersion;
Expand Down Expand Up @@ -659,20 +698,15 @@ private void readMeta( Layout<KEY,VALUE> layout, PagedFile pagedFile ) throws IO
"what it was created with. Created with %d, opened with %d", "what it was created with. Created with %d, opened with %d",
formatVersion, FORMAT_VERSION ); formatVersion, FORMAT_VERSION );
} }
if ( layoutIdentifier != layout.identifier() ) if ( !layout.compatibleWith( layoutIdentifier, majorVersion, minorVersion ) )
{
throw new MetadataMismatchException(
"Tried to open using different layout identifier " +
"than what it was created with. Created with %d, opened with %d",
layoutIdentifier, layout.identifier() );
}
if ( majorVersion != layout.majorVersion() || minorVersion != layout.minorVersion() )
{ {
throw new MetadataMismatchException( throw new MetadataMismatchException(
"Tried to open using different layout version " + "Tried to open using layout not compatible with " +
"than what it was created with. Created with %d.%d, opened with %d.%d", "what the index was created with. Created with: layoutIdentifier=%d,majorVersion=%d,minorVersion=%d. " +
majorVersion, minorVersion, layout.majorVersion(), layout.minorVersion() ); "Opened with layoutIdentifier=%d,majorVersion=%d,minorVersion=%d",
layoutIdentifier, majorVersion, minorVersion, layout.identifier(), layout.majorVersion(), layout.minorVersion() );
} }
return pageSize;
} }


private void writeMeta( Layout<KEY,VALUE> layout, PagedFile pagedFile ) throws IOException private void writeMeta( Layout<KEY,VALUE> layout, PagedFile pagedFile ) throws IOException
Expand All @@ -689,7 +723,7 @@ private void writeMeta( Layout<KEY,VALUE> layout, PagedFile pagedFile ) throws I
} }
} }


private PagedFile mapWithCorrectPageSize( PageCache pageCache, File indexFile, PagedFile pagedFile ) private static PagedFile mapWithCorrectPageSize( PageCache pageCache, File indexFile, PagedFile pagedFile, int pageSize )
throws IOException throws IOException
{ {
// This index was created with another page size, re-open with that actual page size // This index was created with another page size, re-open with that actual page size
Expand Down
111 changes: 109 additions & 2 deletions community/index/src/main/java/org/neo4j/index/internal/gbptree/Layout.java
View file
Expand Up @@ -19,8 +19,10 @@
*/ */
package org.neo4j.index.internal.gbptree; package org.neo4j.index.internal.gbptree;


import java.io.File;
import java.util.Comparator; import java.util.Comparator;


import org.neo4j.io.pagecache.PageCache;
import org.neo4j.io.pagecache.PageCursor; import org.neo4j.io.pagecache.PageCursor;


import static java.lang.String.format; import static java.lang.String.format;
Expand Down Expand Up @@ -161,16 +163,34 @@ static long namedIdentifier( String name, int checksum )
upperInt |= byteValue & 0xFF; upperInt |= byteValue & 0xFF;
} }


return upperInt << Integer.SIZE | (checksum & 0xFFFFFFFF); return (upperInt << Integer.SIZE) | checksum;
} }


/**
* Typically, layout is compatible with given identifier, major and minor version if
* <ul>
* <li>{@code layoutIdentifier == this.identifier()}</li>
* <li>{@code majorVersion == this.majorVersion()}</li>
* <li>{@code minorVersion == this.minorVersion()}</li>
* </ul>
* <p>
* Edge case: When only want to {@link GBPTree#readHeader(PageCache, File, Layout, Header.Reader)} read header} of index and not
* actually use it there may be multiple different combinations that is considered to be valid.
*
* @param layoutIdentifier the stored layout identifier we want to check compatibility against.
* @param majorVersion the stored major version we want to check compatibility against.
* @param minorVersion the stored minor version we want to check compatibility against.
* @return true if this layout is compatible with combination of identifier, major and minor version, false otherwise.
*/
boolean compatibleWith( long layoutIdentifier, int majorVersion, int minorVersion );

/** /**
* Adapter for {@link Layout}, which contains convenient standard implementations of some methods. * Adapter for {@link Layout}, which contains convenient standard implementations of some methods.
* *
* @param <KEY> type of key * @param <KEY> type of key
* @param <VALUE> type of value * @param <VALUE> type of value
*/ */
abstract class Adapter<KEY,VALUE> implements Layout<KEY,VALUE> abstract class Adapter<KEY, VALUE> implements Layout<KEY,VALUE>
{ {
@Override @Override
public String toString() public String toString()
Expand All @@ -179,5 +199,92 @@ public String toString()
getClass().getSimpleName(), majorVersion(), minorVersion(), identifier(), getClass().getSimpleName(), majorVersion(), minorVersion(), identifier(),
keySize(), valueSize() ); keySize(), valueSize() );
} }

@Override
public boolean compatibleWith( long layoutIdentifier, int majorVersion, int minorVersion )
{
return layoutIdentifier == identifier() && majorVersion == majorVersion() && minorVersion == minorVersion();
}
}

abstract class ReadOnlyMetaLayout<KEY, VALUE> implements Layout<KEY,VALUE>
{
@Override
public KEY newKey()
{
throw new UnsupportedOperationException( "Not allowed with read only layout" );
}

@Override
public KEY copyKey( KEY key, KEY into )
{
throw new UnsupportedOperationException( "Not allowed with read only layout" );
}

@Override
public VALUE newValue()
{
throw new UnsupportedOperationException( "Not allowed with read only layout" );
}

@Override
public int keySize()
{
throw new UnsupportedOperationException( "Not allowed with read only layout" );
}

@Override
public int valueSize()
{
throw new UnsupportedOperationException( "Not allowed with read only layout" );
}

@Override
public void writeKey( PageCursor cursor, KEY key )
{
throw new UnsupportedOperationException( "Not allowed with read only layout" );
}

@Override
public void writeValue( PageCursor cursor, VALUE value )
{
throw new UnsupportedOperationException( "Not allowed with read only layout" );
}

@Override
public void readKey( PageCursor cursor, KEY into )
{
throw new UnsupportedOperationException( "Not allowed with read only layout" );
}

@Override
public void readValue( PageCursor cursor, VALUE into )
{
throw new UnsupportedOperationException( "Not allowed with read only layout" );
}

@Override
public long identifier()
{
throw new UnsupportedOperationException( "Not allowed with read only layout" );
}

@Override
public int majorVersion()
{
throw new UnsupportedOperationException( "Not allowed with read only layout" );
}

@Override
public int minorVersion()
{
throw new UnsupportedOperationException( "Not allowed with read only layout" );
}

@Override
public int compare( KEY o1, KEY o2 )
{
throw new UnsupportedOperationException( "Not allowed with read only layout" );
}
} }
} }

0 comments on commit 077d500

Please sign in to comment.