Skip to content

Commit

Permalink
Make CommonAbstractStore check page cursor bounds
Browse files Browse the repository at this point in the history
  • Loading branch information
@chrisvest
chrisvest committed Apr 7, 2016
1 parent 7743afe commit f5aefc6
Show file tree
Hide file tree
Showing 3 changed files with 387 additions and 7 deletions.
71 changes: 65 additions & 6 deletions community/kernel/src/main/java/org/neo4j/kernel/impl/store/CommonAbstractStore.java
View file
Expand Up @@ -204,6 +204,12 @@ protected void initialiseNewStoreFile( PagedFile file ) throws IOException
createHeaderRecord( pageCursor ); createHeaderRecord( pageCursor );
} }
while ( pageCursor.shouldRetry() ); while ( pageCursor.shouldRetry() );
if ( pageCursor.checkAndClearBoundsFlag() )
{
throw new UnderlyingStorageException(
"Out of page bounds when writing header; page size too small: " +
pageCache.pageSize() + " bytes.");
}
} }
} }
} }
Expand All @@ -216,6 +222,27 @@ protected void initialiseNewStoreFile( PagedFile file ) throws IOException
idGeneratorFactory.create( idFileName, getNumberOfReservedLowIds(), true ); idGeneratorFactory.create( idFileName, getNumberOfReservedLowIds(), true );
} }


protected void checkForOutOfBounds( PageCursor pageCursor, long recordId )
{
if ( pageCursor.checkAndClearBoundsFlag() )
{
throwOutOfBoundsException( recordId );
}
}

private void throwOutOfBoundsException( long recordId )
{
RECORD record = newRecord();
record.setId( recordId );
long pageId = pageIdForRecord( recordId );
int offset = offsetForId( recordId );
throw new UnderlyingStorageException(
"Access to record " + record + " went out of bounds of the page. The record size is " +
recordSize + " bytes, and the access was at offset " + offset + " bytes into page " +
pageId + ", and the pages have a capacity of " + storeFile.pageSize() + " bytes. " +
"The mapped store file in question is " + storageFileName.getAbsolutePath() );
}

protected void createHeaderRecord( PageCursor cursor ) throws IOException protected void createHeaderRecord( PageCursor cursor ) throws IOException
{ {
int offset = cursor.getOffset(); int offset = cursor.getOffset();
Expand Down Expand Up @@ -263,6 +290,12 @@ private void extractHeaderRecord() throws IOException
readHeaderAndInitializeRecordFormat( pageCursor ); readHeaderAndInitializeRecordFormat( pageCursor );
} }
while ( pageCursor.shouldRetry() ); while ( pageCursor.shouldRetry() );
if ( pageCursor.checkAndClearBoundsFlag() )
{
throw new UnderlyingStorageException(
"Out of page bounds when reading header; page size too small: " +
pageCache.pageSize() + " bytes.");
}
} }
} }
} }
Expand Down Expand Up @@ -293,8 +326,8 @@ public int getRecordsPerPage()
public byte[] getRawRecordData( long id ) throws IOException public byte[] getRawRecordData( long id ) throws IOException
{ {
byte[] data = new byte[recordSize]; byte[] data = new byte[recordSize];
long pageId = RecordPageLocationCalculator.pageIdForRecord( id, storeFile.pageSize(), recordSize ); long pageId = pageIdForRecord( id );
int offset = RecordPageLocationCalculator.offsetForId( id, storeFile.pageSize(), recordSize ); int offset = offsetForId( id );
try ( PageCursor pageCursor = storeFile.io( pageId, PagedFile.PF_SHARED_READ_LOCK ) ) try ( PageCursor pageCursor = storeFile.io( pageId, PagedFile.PF_SHARED_READ_LOCK ) )
{ {
if ( pageCursor.next() ) if ( pageCursor.next() )
Expand All @@ -305,6 +338,7 @@ public byte[] getRawRecordData( long id ) throws IOException
pageCursor.getBytes( data ); pageCursor.getBytes( data );
} }
while ( pageCursor.shouldRetry() ); while ( pageCursor.shouldRetry() );
checkForOutOfBounds( pageCursor, id );
} }
} }
return data; return data;
Expand Down Expand Up @@ -365,6 +399,7 @@ public boolean isInUse( long id )
recordIsInUse = isInUse( cursor ); recordIsInUse = isInUse( cursor );
} }
while ( cursor.shouldRetry() ); while ( cursor.shouldRetry() );
checkForOutOfBounds( cursor, id );
} }
return recordIsInUse; return recordIsInUse;
} }
Expand Down Expand Up @@ -396,7 +431,7 @@ final void rebuildIdGenerator()
openIdGenerator(); openIdGenerator();


long defraggedCount = 0; long defraggedCount = 0;
boolean fastRebuild = doFastIdGeneratorRebuild(); boolean fastRebuild = isOnlyFastIdGeneratorRebuildEnabled( configuration );


try try
{ {
Expand Down Expand Up @@ -426,6 +461,11 @@ final void rebuildIdGenerator()
} }
} }


protected boolean isOnlyFastIdGeneratorRebuildEnabled( Config config )
{
return config.get( Configuration.rebuild_idgenerators_fast );
}

private long rebuildIdGeneratorSlow( PageCursor cursor, int recordsPerPage, int blockSize, private long rebuildIdGeneratorSlow( PageCursor cursor, int recordsPerPage, int blockSize,
long foundHighId ) long foundHighId )
throws IOException throws IOException
Expand Down Expand Up @@ -469,6 +509,7 @@ else if ( isRecordReserved( cursor ) )
} }
} }
while ( cursor.shouldRetry() ); while ( cursor.shouldRetry() );
checkIdScanCursorBounds( cursor );


for ( int i = 0; i < defragged; i++ ) for ( int i = 0; i < defragged; i++ )
{ {
Expand All @@ -480,9 +521,14 @@ else if ( isRecordReserved( cursor ) )
return defragCount; return defragCount;
} }


protected boolean doFastIdGeneratorRebuild() private void checkIdScanCursorBounds( PageCursor cursor )
{ {
return configuration.get( Configuration.rebuild_idgenerators_fast ); if ( cursor.checkAndClearBoundsFlag() )
{
throw new UnderlyingStorageException(
"Out of bounds access on page " + cursor.getCurrentPageId() + " detected while scanning the " +
storageFileName + " file for deleted records" );
}
} }


/** /**
Expand Down Expand Up @@ -653,11 +699,14 @@ protected long scanForHighId()
long nextPageId = storeFile.getLastPageId(); long nextPageId = storeFile.getLastPageId();
int recordsPerPage = getRecordsPerPage(); int recordsPerPage = getRecordsPerPage();
int recordSize = getRecordSize(); int recordSize = getRecordSize();
long highestId = getNumberOfReservedLowIds();
while ( nextPageId >= 0 && cursor.next( nextPageId ) ) while ( nextPageId >= 0 && cursor.next( nextPageId ) )
{ {
nextPageId--; nextPageId--;
boolean found;
do do
{ {
found = false;
int currentRecord = recordsPerPage; int currentRecord = recordsPerPage;
while ( currentRecord-- > 0 ) while ( currentRecord-- > 0 )
{ {
Expand All @@ -671,12 +720,19 @@ protected long scanForHighId()
if ( !justLegacyStoreTrailer ) if ( !justLegacyStoreTrailer )
{ {
// We've found the highest id in use // We've found the highest id in use
return recordId + 1 /*+1 since we return the high id*/; found = true;
highestId = recordId + 1; /*+1 since we return the high id*/;
break;
} }
} }
} }
} }
while ( cursor.shouldRetry() ); while ( cursor.shouldRetry() );
checkIdScanCursorBounds( cursor );
if ( found )
{
return highestId;
}
} }


return getNumberOfReservedLowIds(); return getNumberOfReservedLowIds();
Expand Down Expand Up @@ -986,6 +1042,7 @@ public RECORD getRecord( long id, RECORD record, RecordLoad mode )
recordFormat.read( record, cursor, mode, recordSize, storeFile ); recordFormat.read( record, cursor, mode, recordSize, storeFile );
} }
while ( cursor.shouldRetry() ); while ( cursor.shouldRetry() );
checkForOutOfBounds( cursor, id );
verifyAfterReading( record, mode ); verifyAfterReading( record, mode );
} }
else else
Expand Down Expand Up @@ -1018,6 +1075,7 @@ public void updateRecord( RECORD record )
recordFormat.write( record, cursor, recordSize, storeFile ); recordFormat.write( record, cursor, recordSize, storeFile );
} }
while ( cursor.shouldRetry() ); while ( cursor.shouldRetry() );
checkForOutOfBounds( cursor, id ); // We don't free ids if something weird goes wrong
if ( !record.inUse() ) if ( !record.inUse() )
{ {
freeId( id ); freeId( id );
Expand Down Expand Up @@ -1112,6 +1170,7 @@ public boolean next()
recordFormat.read( record, pageCursor, mode, recordSize, storeFile ); recordFormat.read( record, pageCursor, mode, recordSize, storeFile );
} }
while ( pageCursor.shouldRetry() ); while ( pageCursor.shouldRetry() );
checkForOutOfBounds( pageCursor, currentId );
verifyAfterReading( record, mode ); verifyAfterReading( record, mode );
} }
else else
Expand Down
2 changes: 1 addition & 1 deletion community/kernel/src/main/java/org/neo4j/kernel/impl/store/TokenStore.java
View file
Expand Up @@ -76,7 +76,7 @@ public DynamicStringStore getNameStore()
} }


@Override @Override
protected boolean doFastIdGeneratorRebuild() protected boolean isOnlyFastIdGeneratorRebuildEnabled( Config config )
{ {
return false; return false;
} }
Expand Down

0 comments on commit f5aefc6

Please sign in to comment.