Skip to content

Commit

Permalink
Removed rangeSize option from Native LSS
Browse files Browse the repository at this point in the history
since the only reasonable option is 64. This simplifies code as well.
  • Loading branch information
tinwelint committed Jan 12, 2017
1 parent 100a6ac commit b031b2b
Show file tree
Hide file tree
Showing 12 changed files with 43 additions and 138 deletions.
Expand Up @@ -122,7 +122,9 @@ public interface Layout<KEY, VALUE> extends Comparator<KEY>
* *
* @param cursor {@link PageCursor} to write into, at its current offset. * @param cursor {@link PageCursor} to write into, at its current offset.
*/ */
void writeMetaData( PageCursor cursor ); default void writeMetaData( PageCursor cursor )
{ // no meta-data by default
}


/** /**
* Reads meta data specific to this layout instance from {@code cursor} at its current offset. * Reads meta data specific to this layout instance from {@code cursor} at its current offset.
Expand All @@ -133,7 +135,9 @@ public interface Layout<KEY, VALUE> extends Comparator<KEY>
* *
* @param cursor {@link PageCursor} to read from, at its current offset. * @param cursor {@link PageCursor} to read from, at its current offset.
*/ */
void readMetaData( PageCursor cursor ); default void readMetaData( PageCursor cursor )
{ // no meta-data by default
}


/** /**
* Utility method for generating an {@link #identifier()}. * Utility method for generating an {@link #identifier()}.
Expand Down
Expand Up @@ -47,6 +47,6 @@ public NativeLabelScanStoreExtension()
public Lifecycle newInstance( KernelContext context, Dependencies dependencies ) throws Throwable public Lifecycle newInstance( KernelContext context, Dependencies dependencies ) throws Throwable
{ {
return new LabelScanStoreProvider( new NativeLabelScanStore( dependencies.pageCache(), return new LabelScanStoreProvider( new NativeLabelScanStore( dependencies.pageCache(),
context.storeDir(), Long.SIZE, new FullLabelStream( dependencies.indexStoreView() ) ), 0 ); context.storeDir(), new FullLabelStream( dependencies.indexStoreView() ) ), 0 );
} }
} }
Expand Up @@ -21,7 +21,7 @@


/** /**
* Keys in {@link LabelScanLayout}, each key consists of {@code labelId} and {@code nodeIdRange}, i.e. * Keys in {@link LabelScanLayout}, each key consists of {@code labelId} and {@code nodeIdRange}, i.e.
* {@code nodeId/rangeSize}, where each range is a small bit set of size {@code rangeSize}. * {@code nodeId/rangeSize}, where each range is a small bit set of size {@link LabelScanValue#RANGE_SIZE}.
*/ */
class LabelScanKey class LabelScanKey
{ {
Expand Down
Expand Up @@ -23,8 +23,6 @@
import org.neo4j.index.gbptree.Layout; import org.neo4j.index.gbptree.Layout;
import org.neo4j.io.pagecache.PageCursor; import org.neo4j.io.pagecache.PageCursor;


import static java.lang.Long.bitCount;

/** /**
* {@link Layout} for {@link GBPTree} used by {@link NativeLabelScanStore}. * {@link Layout} for {@link GBPTree} used by {@link NativeLabelScanStore}.
* *
Expand All @@ -34,8 +32,7 @@
* </li> * </li>
* <li> * <li>
* Each value is a 64-bit bit set (a primitive {@code long}) where each set bit in it represents * Each value is a 64-bit bit set (a primitive {@code long}) where each set bit in it represents
* a node with that label, such that {@code nodeId = nodeIdRange+bitOffset}. Range size (e.g. 64 bits) * a node with that label, such that {@code nodeId = nodeIdRange+bitOffset}. Range size is 64 bits.
* is configurable on initial creation of the store, 8, 16, 32 or 64.
* </li> * </li>
* </ul> * </ul>
*/ */
Expand All @@ -51,26 +48,6 @@ class LabelScanLayout implements Layout<LabelScanKey,LabelScanValue>
*/ */
private static final int KEY_SIZE = Integer.BYTES/*labelId*/ + 6/*idRange*/; private static final int KEY_SIZE = Integer.BYTES/*labelId*/ + 6/*idRange*/;


/**
* Size of each node id range, e.g. 8, 16, 32 or 64. This value is written at creation,
* otherwise verified against index meta data on open.
*/
private final int rangeSize;

/**
* {@link #rangeSize}, as number of bytes (instead of number of bits).
*/
private final int rangeSizeBytes;

LabelScanLayout( int rangeSize )
{
// asserts values are 8, 16, 32 or 64
assert bitCount( rangeSize ) == 1 && (rangeSize & ~0b1111000) == 0;

this.rangeSize = rangeSize;
this.rangeSizeBytes = rangeSize >>> 3;
}

/** /**
* Compares {@link LabelScanKey}, giving ascending order of {@code labelId} then {@code nodeIdRange}. * Compares {@link LabelScanKey}, giving ascending order of {@code labelId} then {@code nodeIdRange}.
*/ */
Expand Down Expand Up @@ -110,7 +87,7 @@ public int keySize()
@Override @Override
public int valueSize() public int valueSize()
{ {
return rangeSizeBytes; return LabelScanValue.RANGE_SIZE_BYTES;
} }


@Override @Override
Expand All @@ -129,23 +106,7 @@ private static void put6ByteLong( PageCursor cursor, long value )
@Override @Override
public void writeValue( PageCursor cursor, LabelScanValue value ) public void writeValue( PageCursor cursor, LabelScanValue value )
{ {
switch ( rangeSize ) cursor.putLong( value.bits );
{
case 8:
cursor.putByte( (byte) value.bits );
break;
case 16:
cursor.putShort( (short) value.bits );
break;
case 32:
cursor.putInt( (int) value.bits );
break;
case 64:
cursor.putLong( value.bits );
break;
default:
throw new IllegalArgumentException( String.valueOf( rangeSize ) );
}
} }


@Override @Override
Expand All @@ -165,29 +126,13 @@ private static long get6ByteLong( PageCursor cursor )
@Override @Override
public void readValue( PageCursor cursor, LabelScanValue into ) public void readValue( PageCursor cursor, LabelScanValue into )
{ {
switch ( rangeSize ) into.bits = cursor.getLong();
{
case 8:
into.bits = cursor.getByte() & 0xFF;
break;
case 16:
into.bits = cursor.getShort() & 0xFFFF;
break;
case 32:
into.bits = cursor.getInt() & 0xFFFFFFFFL;
break;
case 64:
into.bits = cursor.getLong();
break;
default:
throw new IllegalArgumentException( String.valueOf( rangeSize ) );
}
} }


@Override @Override
public long identifier() public long identifier()
{ {
return Layout.namedIdentifier( IDENTIFIER_NAME, rangeSize ); return Layout.namedIdentifier( IDENTIFIER_NAME, LabelScanValue.RANGE_SIZE );
} }


@Override @Override
Expand All @@ -201,27 +146,4 @@ public int minorVersion()
{ {
return 1; return 1;
} }

/**
* Writes node id range size to the index meta data. The range size cannot change.
*/
@Override
public void writeMetaData( PageCursor cursor )
{
cursor.putInt( rangeSize );
}

/**
* Reads node id range size of the index.
*/
@Override
public void readMetaData( PageCursor cursor )
{
int rangeSize = cursor.getInt();
if ( this.rangeSize != rangeSize )
{
throw new IllegalArgumentException( "A different range size " + this.rangeSize +
" was specified when loading an index with actual range size " + rangeSize );
}
}
} }
Expand Up @@ -24,6 +24,9 @@
*/ */
class LabelScanValue class LabelScanValue
{ {
static final int RANGE_SIZE = Long.SIZE;
static final int RANGE_SIZE_BYTES = Long.BYTES;

/** /**
* Small bit set. * Small bit set.
*/ */
Expand Down
Expand Up @@ -35,11 +35,6 @@
*/ */
class LabelScanValueIterator extends PrimitiveLongCollections.PrimitiveLongBaseIterator class LabelScanValueIterator extends PrimitiveLongCollections.PrimitiveLongBaseIterator
{ {
/**
* Size of nodeId ranges, i.e. how many node ids one {@link LabelScanValue} can hold at most.
*/
private final int rangeSize;

/** /**
* {@link RawCursor} to lazily read new {@link LabelScanValue} from. * {@link RawCursor} to lazily read new {@link LabelScanValue} from.
*/ */
Expand All @@ -65,9 +60,8 @@ class LabelScanValueIterator extends PrimitiveLongCollections.PrimitiveLongBaseI
*/ */
private long prevRange = -1; private long prevRange = -1;


LabelScanValueIterator( int rangeSize, RawCursor<Hit<LabelScanKey,LabelScanValue>,IOException> cursor ) LabelScanValueIterator( RawCursor<Hit<LabelScanKey,LabelScanValue>,IOException> cursor )
{ {
this.rangeSize = rangeSize;
this.cursor = cursor; this.cursor = cursor;
} }


Expand Down Expand Up @@ -99,7 +93,7 @@ protected boolean fetchNext()
} }


Hit<LabelScanKey,LabelScanValue> hit = cursor.get(); Hit<LabelScanKey,LabelScanValue> hit = cursor.get();
baseNodeId = hit.key().idRange * rangeSize; baseNodeId = hit.key().idRange * LabelScanValue.RANGE_SIZE;
bits = hit.value().bits; bits = hit.value().bits;


assert keysInOrder( hit.key() ); assert keysInOrder( hit.key() );
Expand Down
Expand Up @@ -47,21 +47,15 @@ class NativeLabelScanReader implements LabelScanReader
*/ */
private final Index<LabelScanKey,LabelScanValue> index; private final Index<LabelScanKey,LabelScanValue> index;


/**
* Size of each nodeId range, i.e. how many potential node ids each {@link LabelScanValue} contains.
*/
private final int rangeSize;

/** /**
* Currently open {@link RawCursor} from query methods below. Open cursors are closed when calling * Currently open {@link RawCursor} from query methods below. Open cursors are closed when calling
* new query methods or when {@link #close() closing} this reader. * new query methods or when {@link #close() closing} this reader.
*/ */
private final Queue<RawCursor<Hit<LabelScanKey,LabelScanValue>,IOException>> openCursors; private final Queue<RawCursor<Hit<LabelScanKey,LabelScanValue>,IOException>> openCursors;


NativeLabelScanReader( Index<LabelScanKey,LabelScanValue> index, int rangeSize ) NativeLabelScanReader( Index<LabelScanKey,LabelScanValue> index )
{ {
this.index = index; this.index = index;
this.rangeSize = rangeSize;
this.openCursors = new LinkedList<>(); this.openCursors = new LinkedList<>();
} }


Expand Down Expand Up @@ -96,7 +90,7 @@ public PrimitiveLongIterator nodesWithLabel( int labelId )
throw new RuntimeException( e ); throw new RuntimeException( e );
} }


return new LabelScanValueIterator( rangeSize, cursor ); return new LabelScanValueIterator( cursor );
} }


@Override @Override
Expand All @@ -123,7 +117,7 @@ private List<PrimitiveLongIterator> iteratorsForLabels( int[] labelIds )
{ {
RawCursor<Hit<LabelScanKey,LabelScanValue>,IOException> cursor = seekerForLabel( labelId ); RawCursor<Hit<LabelScanKey,LabelScanValue>,IOException> cursor = seekerForLabel( labelId );
openCursors.offer( cursor ); openCursors.offer( cursor );
iterators.add( new LabelScanValueIterator( rangeSize, cursor ) ); iterators.add( new LabelScanValueIterator( cursor ) );
} }
} }
catch ( IOException e ) catch ( IOException e )
Expand Down
Expand Up @@ -77,12 +77,6 @@ public class NativeLabelScanStore implements LabelScanStore
*/ */
private final File storeFile; private final File storeFile;


/**
* Size of each node id range, e.g. 8, 16, 32 or 64. This value is written at creation,
* otherwise verified against index meta data on open.
*/
private final int rangeSize;

/** /**
* Used in {@link #start()} if the store is empty, where this will provide all data for fully populating * Used in {@link #start()} if the store is empty, where this will provide all data for fully populating
* this label scan store. This can be the case when changing label scan store provider on an existing database. * this label scan store. This can be the case when changing label scan store provider on an existing database.
Expand Down Expand Up @@ -114,24 +108,23 @@ public class NativeLabelScanStore implements LabelScanStore


private final NativeLabelScanWriter singleWriter; private final NativeLabelScanWriter singleWriter;


public NativeLabelScanStore( PageCache pageCache, File storeDir, int rangeSize, public NativeLabelScanStore( PageCache pageCache, File storeDir,
FullStoreChangeStream fullStoreChangeStream ) FullStoreChangeStream fullStoreChangeStream )
{ {
this( pageCache, storeDir, rangeSize, fullStoreChangeStream, 0/*means no opinion about page size*/ ); this( pageCache, storeDir, fullStoreChangeStream, 0/*means no opinion about page size*/ );
} }


/* /*
* Test access to be able to control page size. * Test access to be able to control page size.
*/ */
NativeLabelScanStore( PageCache pageCache, File storeDir, int rangeSize, NativeLabelScanStore( PageCache pageCache, File storeDir,
FullStoreChangeStream fullStoreChangeStream, int pageSize ) FullStoreChangeStream fullStoreChangeStream, int pageSize )
{ {
this.pageCache = pageCache; this.pageCache = pageCache;
this.pageSize = pageSize; this.pageSize = pageSize;
this.fullStoreChangeStream = fullStoreChangeStream; this.fullStoreChangeStream = fullStoreChangeStream;
this.storeFile = new File( storeDir, DEFAULT_NAME + ".labelscanstore.db" ); this.storeFile = new File( storeDir, DEFAULT_NAME + ".labelscanstore.db" );
this.rangeSize = rangeSize; this.singleWriter = new NativeLabelScanWriter( 1_000 );
this.singleWriter = new NativeLabelScanWriter( rangeSize, 1_000 );
} }


/** /**
Expand All @@ -143,7 +136,7 @@ public NativeLabelScanStore( PageCache pageCache, File storeDir, int rangeSize,
@Override @Override
public LabelScanReader newReader() public LabelScanReader newReader()
{ {
return new NativeLabelScanReader( index, rangeSize ); return new NativeLabelScanReader( index );
} }


/** /**
Expand Down Expand Up @@ -226,7 +219,7 @@ public ResourceIterator<File> snapshotStoreFiles() throws IOException
@Override @Override
public void init() throws IOException public void init() throws IOException
{ {
index = new GBPTree<>( pageCache, storeFile, new LabelScanLayout( rangeSize ), pageSize, NO_MONITOR ); index = new GBPTree<>( pageCache, storeFile, new LabelScanLayout(), pageSize, NO_MONITOR );
} }


/** /**
Expand Down
Expand Up @@ -33,6 +33,8 @@
import static java.lang.Long.min; import static java.lang.Long.min;
import static java.lang.Math.toIntExact; import static java.lang.Math.toIntExact;


import static org.neo4j.kernel.impl.index.labelscan.LabelScanValue.RANGE_SIZE;

/** /**
* {@link LabelScanWriter} for {@link NativeLabelScanStore}, or rather an {@link IndexWriter} for its * {@link LabelScanWriter} for {@link NativeLabelScanStore}, or rather an {@link IndexWriter} for its
* internal {@link GBPTree}. * internal {@link GBPTree}.
Expand Down Expand Up @@ -84,12 +86,6 @@ class NativeLabelScanWriter implements LabelScanWriter
*/ */
private final LabelScanValue value = new LabelScanValue(); private final LabelScanValue value = new LabelScanValue();


/**
* Configured range size for this index. Range size is the size of each {@link LabelScanValue},
* i.e. how big each bit set range is, e.g. 8, 16, 32 or 64.
*/
private final int rangeSize;

/** /**
* Batch currently building up as {@link #write(NodeLabelUpdate) updates} come in. Cursor for where * Batch currently building up as {@link #write(NodeLabelUpdate) updates} come in. Cursor for where
* to place new updates is {@link #pendingUpdatesCursor}. Length of this queue is decided in constructor * to place new updates is {@link #pendingUpdatesCursor}. Length of this queue is decided in constructor
Expand Down Expand Up @@ -120,9 +116,8 @@ class NativeLabelScanWriter implements LabelScanWriter
*/ */
private long lowestLabelId; private long lowestLabelId;


NativeLabelScanWriter( int rangeSize, int batchSize ) NativeLabelScanWriter( int batchSize )
{ {
this.rangeSize = rangeSize;
this.pendingUpdates = new NodeLabelUpdate[batchSize]; this.pendingUpdates = new NodeLabelUpdate[batchSize];
} }


Expand Down Expand Up @@ -241,7 +236,7 @@ private void change( long currentLabelId, long nodeId, boolean add ) throws IOEx
addition = add; addition = add;
} }


value.set( toIntExact( nodeId % rangeSize ) ); value.set( toIntExact( nodeId % RANGE_SIZE ) );
} }


private void flushPendingRange() throws IOException private void flushPendingRange() throws IOException
Expand All @@ -256,9 +251,9 @@ private void flushPendingRange() throws IOException
} }
} }


private long rangeOf( long nodeId ) private static long rangeOf( long nodeId )
{ {
return nodeId / rangeSize; return nodeId / RANGE_SIZE;
} }


/** /**
Expand Down
Expand Up @@ -55,16 +55,19 @@ public void shouldFindMultipleNodesInEachRange() throws Exception
null ); null );
when( index.seek( any( LabelScanKey.class ), any( LabelScanKey.class ) ) ) when( index.seek( any( LabelScanKey.class ), any( LabelScanKey.class ) ) )
.thenReturn( cursor ); .thenReturn( cursor );
try ( NativeLabelScanReader reader = new NativeLabelScanReader( index, 16 ) ) try ( NativeLabelScanReader reader = new NativeLabelScanReader( index ) )
{ {
// WHEN // WHEN
PrimitiveLongIterator iterator = reader.nodesWithLabel( LABEL_ID ); PrimitiveLongIterator iterator = reader.nodesWithLabel( LABEL_ID );


// THEN // THEN
assertArrayEquals( new long[] { assertArrayEquals( new long[] {
// base 0*64 = 0
1, 6, 7, 11, 15, 1, 6, 7, 11, 15,
19, 25, // base 1*64 = 64
48, 53, 55, 61 }, 64 + 3, 64 + 9,
// base 3*64 = 192
192 + 0, 192 + 5, 192 + 7, 192 + 13 },


asArray( iterator ) ); asArray( iterator ) );
} }
Expand Down

0 comments on commit b031b2b

Please sign in to comment.