Skip to content

Commit

Permalink
Moved highId responsibility to IdGenerator
Browse files Browse the repository at this point in the history
  • Loading branch information
@klaren @MishaDemianenko
klaren authored and MishaDemianenko committed Jul 4, 2017
1 parent 6a18fe0 commit 46b4df5
Show file tree
Hide file tree
Showing 5 changed files with 109 additions and 150 deletions.
17 changes: 7 additions & 10 deletions community/kernel/src/main/java/org/neo4j/kernel/impl/store/id/FreeIdKeeper.java
View file
Expand Up @@ -27,6 +27,8 @@
import org.neo4j.io.fs.StoreChannel; import org.neo4j.io.fs.StoreChannel;
import org.neo4j.kernel.impl.store.UnderlyingStorageException; import org.neo4j.kernel.impl.store.UnderlyingStorageException;


import static org.neo4j.kernel.impl.store.id.IdFile.NO_RESULT;

/** /**
* Instances of this class maintain a list of free ids with the potential to overflow to disk if the number * Instances of this class maintain a list of free ids with the potential to overflow to disk if the number
* of free ids becomes too large. * of free ids becomes too large.
Expand All @@ -41,8 +43,8 @@
*/ */
public class FreeIdKeeper implements Closeable public class FreeIdKeeper implements Closeable
{ {
public static final long NO_RESULT = -1;
public static final int ID_ENTRY_SIZE = Long.BYTES; private static final int ID_ENTRY_SIZE = Long.BYTES;
private final LinkedList<Long> freeIds = new LinkedList<>(); private final LinkedList<Long> freeIds = new LinkedList<>();
private final LinkedList<Long> readFromDisk = new LinkedList<>(); private final LinkedList<Long> readFromDisk = new LinkedList<>();
private final StoreChannel channel; private final StoreChannel channel;
Expand All @@ -66,22 +68,17 @@ public class FreeIdKeeper implements Closeable
private long maxReadPosition; private long maxReadPosition;
private long readPosition; // the place from where we read. Always <= maxReadPosition private long readPosition; // the place from where we read. Always <= maxReadPosition


public FreeIdKeeper( StoreChannel channel, int threshold, boolean aggressiveReuse ) throws IOException public FreeIdKeeper( StoreChannel channel, int threshold, boolean aggressiveReuse, int initialPosition ) throws IOException
{ {
this.channel = channel; this.channel = channel;
this.threshold = threshold; this.threshold = threshold;
this.aggressiveReuse = aggressiveReuse; this.aggressiveReuse = aggressiveReuse;
this.lowWatermarkForChannelPosition = channel.position(); this.lowWatermarkForChannelPosition = initialPosition;
readPosition = lowWatermarkForChannelPosition; readPosition = lowWatermarkForChannelPosition;
restoreIdsOnStartup();
}


private void restoreIdsOnStartup() throws IOException
{
// this is always true regardless of aggressiveReuse. It only matters once we start writing // this is always true regardless of aggressiveReuse. It only matters once we start writing
maxReadPosition = channel.size(); maxReadPosition = this.channel.size();
defraggedIdCount = ( maxReadPosition - lowWatermarkForChannelPosition ) / ID_ENTRY_SIZE; defraggedIdCount = ( maxReadPosition - lowWatermarkForChannelPosition ) / ID_ENTRY_SIZE;
readIdBatch();
} }


public void freeId( long id ) public void freeId( long id )
Expand Down
41 changes: 0 additions & 41 deletions community/kernel/src/main/java/org/neo4j/kernel/impl/store/id/HighIdKeeper.java
View file

This file was deleted.

97 changes: 38 additions & 59 deletions community/kernel/src/main/java/org/neo4j/kernel/impl/store/id/IdFile.java
View file
Expand Up @@ -19,7 +19,6 @@
*/ */
package org.neo4j.kernel.impl.store.id; package org.neo4j.kernel.impl.store.id;


import java.io.Closeable;
import java.io.File; import java.io.File;
import java.io.IOException; import java.io.IOException;
import java.nio.ByteBuffer; import java.nio.ByteBuffer;
Expand All @@ -29,11 +28,11 @@
import org.neo4j.kernel.impl.store.InvalidIdGeneratorException; import org.neo4j.kernel.impl.store.InvalidIdGeneratorException;
import org.neo4j.kernel.impl.store.UnderlyingStorageException; import org.neo4j.kernel.impl.store.UnderlyingStorageException;


import static java.lang.Math.max;



public class IdFile
public class IdFile implements Closeable
{ {
public static final long NO_RESULT = -1;

// sticky(byte), nextFreeId(long) // sticky(byte), nextFreeId(long)
public static final int HEADER_SIZE = Byte.BYTES + Long.BYTES; public static final int HEADER_SIZE = Byte.BYTES + Long.BYTES;


Expand All @@ -47,14 +46,15 @@ public class IdFile implements Closeable
private StoreChannel fileChannel; private StoreChannel fileChannel;


private FreeIdKeeper freeIdKeeper; private FreeIdKeeper freeIdKeeper;
private final HighIdKeeper highIdKeeper = new HighIdKeeper();


private final int grabSize; private final int grabSize;
private final boolean aggressiveReuse; private final boolean aggressiveReuse;


private long initialHighId;

private boolean closed = true; private boolean closed = true;


public IdFile( FileSystemAbstraction fs, File file, int grabSize, boolean aggressiveReuse, long highId ) public IdFile( FileSystemAbstraction fs, File file, int grabSize, boolean aggressiveReuse )
{ {
if ( grabSize < 1 ) if ( grabSize < 1 )
{ {
Expand All @@ -65,7 +65,6 @@ public IdFile( FileSystemAbstraction fs, File file, int grabSize, boolean aggres
this.fs = fs; this.fs = fs;
this.grabSize = grabSize; this.grabSize = grabSize;
this.aggressiveReuse = aggressiveReuse; this.aggressiveReuse = aggressiveReuse;
highIdKeeper.setHighId( highId );
} }


// initialize the id generator and performs a simple validation // initialize the id generator and performs a simple validation
Expand All @@ -74,13 +73,10 @@ void init()
try try
{ {
fileChannel = fs.open( file, "rw" ); fileChannel = fs.open( file, "rw" );
initialHighId = readAndValidateHeader();
markAsSticky();


ByteBuffer buffer = readHeader(); this.freeIdKeeper = new FreeIdKeeper( fileChannel, grabSize, aggressiveReuse, HEADER_SIZE );
highIdKeeper.setHighId( max( buffer.getLong(), highIdKeeper.getHighId() ) );
markAsSticky( buffer );

fileChannel.position( HEADER_SIZE );
this.freeIdKeeper = new FreeIdKeeper( fileChannel, grabSize, aggressiveReuse );
closed = false; closed = false;
} }
catch ( IOException e ) catch ( IOException e )
Expand All @@ -95,6 +91,11 @@ public boolean isClosed()
return closed; return closed;
} }


public long getInitialHighId()
{
return initialHighId;
}

void assertStillOpen() void assertStillOpen()
{ {
if ( closed ) if ( closed )
Expand All @@ -103,13 +104,11 @@ void assertStillOpen()
} }
} }


private ByteBuffer readHeader() throws IOException private long readAndValidateHeader() throws IOException
{ {
try try
{ {
ByteBuffer buffer = readHighIdFromHeader( fileChannel, file ); return readAndValidate( fileChannel, file );

return buffer;
} }
catch ( InvalidIdGeneratorException e ) catch ( InvalidIdGeneratorException e )
{ {
Expand All @@ -118,7 +117,7 @@ private ByteBuffer readHeader() throws IOException
} }
} }


private static ByteBuffer readHighIdFromHeader( StoreChannel channel, File fileName ) throws IOException private static long readAndValidate( StoreChannel channel, File fileName ) throws IOException
{ {
ByteBuffer buffer = ByteBuffer.allocate( HEADER_SIZE ); ByteBuffer buffer = ByteBuffer.allocate( HEADER_SIZE );
int read = channel.read( buffer ); int read = channel.read( buffer );
Expand All @@ -134,32 +133,36 @@ private static ByteBuffer readHighIdFromHeader( StoreChannel channel, File fileN
throw new InvalidIdGeneratorException( "Sticky generator[ " + throw new InvalidIdGeneratorException( "Sticky generator[ " +
fileName + "] delete this id file and build a new one" ); fileName + "] delete this id file and build a new one" );
} }
return buffer; return buffer.getLong();
} }


public static long readHighId( FileSystemAbstraction fileSystem, File file ) throws IOException public static long readHighId( FileSystemAbstraction fileSystem, File file ) throws IOException
{ {
try ( StoreChannel channel = fileSystem.open( file, "r" ) ) try ( StoreChannel channel = fileSystem.open( file, "r" ) )
{ {
return readHighIdFromHeader( channel, file ).getLong(); return readAndValidate( channel, file );
} }
} }


/** private void markAsSticky() throws IOException
* Made available for testing purposes.
* Marks an id generator as sticky, i.e. not cleanly shut down.
*/
public void markAsSticky( ByteBuffer buffer ) throws IOException
{ {
buffer.clear(); ByteBuffer buffer = ByteBuffer.allocate( Byte.BYTES );
buffer.put( STICKY_GENERATOR ).limit( 1 ).flip(); buffer.put( STICKY_GENERATOR ).flip();
fileChannel.position( 0 ); fileChannel.position( 0 );
fileChannel.write( buffer ); fileChannel.write( buffer );
fileChannel.force( false ); fileChannel.force( false );
} }


@Override private void markAsCleanlyClosed( ) throws IOException
public void close() {
// remove sticky
ByteBuffer buffer = ByteBuffer.allocate( Byte.BYTES );
buffer.put( CLEAN_GENERATOR ).flip();
fileChannel.position( 0 );
fileChannel.write( buffer );
}

public void close( long highId )
{ {
if ( closed ) if ( closed )
{ {
Expand All @@ -169,11 +172,10 @@ public void close()
try try
{ {
freeIdKeeper.close(); // first write out free ids, then mark as clean freeIdKeeper.close(); // first write out free ids, then mark as clean
ByteBuffer buffer = ByteBuffer.allocate( HEADER_SIZE ); writeHeader( highId );
writeHeader( buffer );
fileChannel.force( false ); fileChannel.force( false );


markAsCleanlyClosed( buffer ); markAsCleanlyClosed( );


closeChannel(); closeChannel();
} }
Expand All @@ -191,26 +193,13 @@ private void closeChannel() throws IOException
fileChannel.close(); fileChannel.close();
fileChannel = null; fileChannel = null;
closed = true; closed = true;
// make this generator unusable
highIdKeeper.setHighId( -1L );
}

private void markAsCleanlyClosed( ByteBuffer buffer ) throws IOException
{
// remove sticky
buffer.clear();
buffer.put( CLEAN_GENERATOR );
buffer.limit( 1 );
buffer.flip();
fileChannel.position( 0 );
fileChannel.write( buffer );
} }


private void writeHeader( ByteBuffer buffer ) throws IOException private void writeHeader( long highId ) throws IOException
{ {
ByteBuffer buffer = ByteBuffer.allocate( HEADER_SIZE );
buffer.put( STICKY_GENERATOR ).putLong( highId ).flip();
fileChannel.position( 0 ); fileChannel.position( 0 );
buffer.put( STICKY_GENERATOR ).putLong( highIdKeeper.getHighId() );
buffer.flip();
fileChannel.write( buffer ); fileChannel.write( buffer );
} }


Expand Down Expand Up @@ -243,16 +232,6 @@ public long getReuseableId()
return freeIdKeeper.getId(); return freeIdKeeper.getId();
} }


public long getHighId()
{
return highIdKeeper.getHighId();
}

public void setHighId( long highId )
{
highIdKeeper.setHighId( highId );
}

public void freeId( long id ) public void freeId( long id )
{ {
freeIdKeeper.freeId( id ); freeIdKeeper.freeId( id );
Expand Down Expand Up @@ -307,7 +286,7 @@ public static void createEmptyIdFile( FileSystemAbstraction fs, File fileName, l
public String toString() public String toString()
{ {
return "IdFile{" + "file=" + file + ", fs=" + fs + ", fileChannel=" + fileChannel + ", defragCount=" + return "IdFile{" + "file=" + file + ", fs=" + fs + ", fileChannel=" + fileChannel + ", defragCount=" +
freeIdKeeper.getCount() + ", highIdKeeper=" + highIdKeeper + ", grabSize=" + grabSize + ", aggressiveReuse=" + freeIdKeeper.getCount() + ", grabSize=" + grabSize + ", aggressiveReuse=" +
aggressiveReuse + ", closed=" + closed + '}'; aggressiveReuse + ", closed=" + closed + '}';
} }
} }

0 comments on commit 46b4df5

Please sign in to comment.