Skip to content

Commit

Permalink
Address PR comments
Browse files Browse the repository at this point in the history
  • Loading branch information
MishaDemianenko committed Oct 3, 2017
1 parent 228b054 commit a4307ba
Show file tree
Hide file tree
Showing 5 changed files with 32 additions and 31 deletions.
Expand Up @@ -32,22 +32,23 @@
import org.neo4j.io.fs.FileSystemAbstraction; import org.neo4j.io.fs.FileSystemAbstraction;
import org.neo4j.io.fs.StoreChannel; import org.neo4j.io.fs.StoreChannel;
import org.neo4j.kernel.impl.transaction.log.LogPosition; import org.neo4j.kernel.impl.transaction.log.LogPosition;
import org.neo4j.kernel.impl.transaction.log.PhysicalLogFile;
import org.neo4j.kernel.impl.transaction.log.PhysicalLogFiles; import org.neo4j.kernel.impl.transaction.log.PhysicalLogFiles;


import static java.lang.String.format; import static java.lang.String.format;


/** /**
* Transaction log truncator used during recovery to truncate all the logs after some specified position, that * Transaction log truncator used during recovery to truncate all the logs after some specified position, that
* recovery threats as corrupted or non-readable. * recovery treats as corrupted or non-readable.
* Transaction log file specified by provided log position will be truncated to provided length, any * Transaction log file specified by provided log position will be truncated to provided length, any
* subsequent files will be removed. * subsequent files will be removed.
* Any removed or modified log content will be stored in separate corruption logs archive for further analysis and as * Any removed or modified log content will be stored in separate corruption logs archive for further analysis and as
* an additional safety option to have the possibility to fully restore original logs in a faulty case. * an additional safety option to have the possibility to fully restore original logs in a faulty case.
*/ */
public class CorruptedLogsTruncator public class CorruptedLogsTruncator
{ {
public static final String CORRUPTED_TX_LOGS_FOLDER_NAME = "corrupted-tx-logs"; public static final String CORRUPTED_TX_LOGS_BASE_NAME = "corrupted-" + PhysicalLogFile.DEFAULT_NAME;
private static final String LOG_FILE_ARCHIVE_PATTERN = "corrupted-logs-%d-%d-%d.zip"; private static final String LOG_FILE_ARCHIVE_PATTERN = CORRUPTED_TX_LOGS_BASE_NAME + "-%d-%d-%d.zip";


private final File storeDir; private final File storeDir;
private final PhysicalLogFiles logFiles; private final PhysicalLogFiles logFiles;
Expand All @@ -72,7 +73,7 @@ public void truncate( LogPosition positionAfterLastRecoveredTransaction ) throws
long recoveredTransactionLogVersion = positionAfterLastRecoveredTransaction.getLogVersion(); long recoveredTransactionLogVersion = positionAfterLastRecoveredTransaction.getLogVersion();
long recoveredTransactionOffset = positionAfterLastRecoveredTransaction.getByteOffset(); long recoveredTransactionOffset = positionAfterLastRecoveredTransaction.getByteOffset();
if ( isRecoveredLogCorrupted( recoveredTransactionLogVersion, recoveredTransactionOffset ) || if ( isRecoveredLogCorrupted( recoveredTransactionLogVersion, recoveredTransactionOffset ) ||
isNotLastLogFile( recoveredTransactionLogVersion ) ) haveMoreRecentLogFiles( recoveredTransactionLogVersion ) )
{ {
backupCorruptedContent( recoveredTransactionLogVersion, recoveredTransactionOffset ); backupCorruptedContent( recoveredTransactionLogVersion, recoveredTransactionOffset );
truncateLogFiles( recoveredTransactionLogVersion, recoveredTransactionOffset ); truncateLogFiles( recoveredTransactionLogVersion, recoveredTransactionOffset );
Expand Down Expand Up @@ -124,7 +125,7 @@ private void backupCorruptedContent( long recoveredTransactionLogVersion, long r
private File getArchiveFile( long recoveredTransactionLogVersion, long recoveredTransactionOffset ) private File getArchiveFile( long recoveredTransactionLogVersion, long recoveredTransactionOffset )
throws IOException throws IOException
{ {
File corruptedLogsFolder = new File( storeDir, CORRUPTED_TX_LOGS_FOLDER_NAME ); File corruptedLogsFolder = new File( storeDir, CORRUPTED_TX_LOGS_BASE_NAME );
fs.mkdirs( corruptedLogsFolder ); fs.mkdirs( corruptedLogsFolder );
return new File( corruptedLogsFolder, return new File( corruptedLogsFolder,
format( LOG_FILE_ARCHIVE_PATTERN, recoveredTransactionLogVersion, recoveredTransactionOffset, format( LOG_FILE_ARCHIVE_PATTERN, recoveredTransactionLogVersion, recoveredTransactionOffset,
Expand Down Expand Up @@ -155,7 +156,7 @@ private void copyTransactionLogContent( long logFileIndex, long logOffset, ZipOu
destination.closeEntry(); destination.closeEntry();
} }


private boolean isNotLastLogFile( long recoveredTransactionLogVersion ) private boolean haveMoreRecentLogFiles( long recoveredTransactionLogVersion )
{ {
return logFiles.getHighestLogVersion() > recoveredTransactionLogVersion; return logFiles.getHighestLogVersion() > recoveredTransactionLogVersion;
} }
Expand Down
Expand Up @@ -189,7 +189,6 @@ protected LogTailInformation checkpointTailInformation( long highestLogVersion,
protected ExtractedTransactionRecord extractFirstTxIdAfterPosition( LogPosition initialPosition, long maxLogVersion ) throws IOException protected ExtractedTransactionRecord extractFirstTxIdAfterPosition( LogPosition initialPosition, long maxLogVersion ) throws IOException
{ {
LogPosition currentPosition = initialPosition; LogPosition currentPosition = initialPosition;
ExtractedTransactionRecord transactionRecord = new ExtractedTransactionRecord();
while ( currentPosition.getLogVersion() <= maxLogVersion ) while ( currentPosition.getLogVersion() <= maxLogVersion )
{ {
LogVersionedStoreChannel storeChannel = PhysicalLogFile.tryOpenForVersion( logFiles, fileSystem, LogVersionedStoreChannel storeChannel = PhysicalLogFile.tryOpenForVersion( logFiles, fileSystem,
Expand All @@ -207,17 +206,15 @@ protected ExtractedTransactionRecord extractFirstTxIdAfterPosition( LogPosition
LogEntry entry = cursor.get(); LogEntry entry = cursor.get();
if ( entry instanceof LogEntryCommit ) if ( entry instanceof LogEntryCommit )
{ {
transactionRecord.setId( ((LogEntryCommit) entry).getTxId() ); return new ExtractedTransactionRecord( ((LogEntryCommit) entry).getTxId() );
return transactionRecord;
} }
} }
} }
} }
catch ( Throwable t ) catch ( Throwable t )
{ {
monitor.corruptedLogFile( currentPosition.getLogVersion(), t ); monitor.corruptedLogFile( currentPosition.getLogVersion(), t );
transactionRecord.setFailure( true ); return new ExtractedTransactionRecord( true );
return transactionRecord;
} }
finally finally
{ {
Expand All @@ -227,7 +224,7 @@ protected ExtractedTransactionRecord extractFirstTxIdAfterPosition( LogPosition


currentPosition = LogPosition.start( currentPosition.getLogVersion() + 1 ); currentPosition = LogPosition.start( currentPosition.getLogVersion() + 1 );
} }
return transactionRecord; return new ExtractedTransactionRecord();
} }


/** /**
Expand Down Expand Up @@ -260,33 +257,38 @@ public LogTailInformation getTailInformation() throws UnderlyingStorageException


static class ExtractedTransactionRecord static class ExtractedTransactionRecord
{ {
private long id; private final long id;
private boolean failure; private final boolean failure;


ExtractedTransactionRecord() ExtractedTransactionRecord()
{ {
id = NO_TRANSACTION_ID; this( NO_TRANSACTION_ID, false );
failure = false;
} }


public long getId() ExtractedTransactionRecord( long txId )
{ {
return id; this( txId, false );
} }


public void setId( long id ) ExtractedTransactionRecord( boolean failure )
{ {
this.id = id; this( NO_TRANSACTION_ID, failure );
} }


public boolean isFailure() private ExtractedTransactionRecord( long txId, boolean failure )
{ {
return failure; this.id = txId;
this.failure = failure;
} }


public void setFailure( boolean failure ) public long getId()
{ {
this.failure = failure; return id;
}

public boolean isFailure()
{
return failure;
} }
} }


Expand Down
Expand Up @@ -397,7 +397,7 @@ public void repetitiveRecoveryIfCorruptedLogsWithCheckpoints() throws IOExceptio
assertThat( numberOfRecoveredTransactions, Matchers.greaterThanOrEqualTo( 0 ) ); assertThat( numberOfRecoveredTransactions, Matchers.greaterThanOrEqualTo( 0 ) );
} }


File corruptedLogArchives = new File( storeDir, CorruptedLogsTruncator.CORRUPTED_TX_LOGS_FOLDER_NAME ); File corruptedLogArchives = new File( storeDir, CorruptedLogsTruncator.CORRUPTED_TX_LOGS_BASE_NAME );
assertThat( corruptedLogArchives.listFiles(), not( emptyArray() ) ); assertThat( corruptedLogArchives.listFiles(), not( emptyArray() ) );
} }


Expand Down
Expand Up @@ -108,7 +108,7 @@ public void pruneAndArchiveLastLog() throws IOException
assertEquals( TOTAL_NUMBER_OF_LOG_FILES, storeDir.listFiles( LogFiles.FILENAME_FILTER ).length ); assertEquals( TOTAL_NUMBER_OF_LOG_FILES, storeDir.listFiles( LogFiles.FILENAME_FILTER ).length );
assertEquals( byteOffset, highestLogFile.length() ); assertEquals( byteOffset, highestLogFile.length() );


File corruptedLogsDirectory = new File( storeDir, CorruptedLogsTruncator.CORRUPTED_TX_LOGS_FOLDER_NAME ); File corruptedLogsDirectory = new File( storeDir, CorruptedLogsTruncator.CORRUPTED_TX_LOGS_BASE_NAME );
assertTrue( corruptedLogsDirectory.exists() ); assertTrue( corruptedLogsDirectory.exists() );
File[] files = corruptedLogsDirectory.listFiles(); File[] files = corruptedLogsDirectory.listFiles();
assertEquals( 1, files.length ); assertEquals( 1, files.length );
Expand Down Expand Up @@ -139,7 +139,7 @@ public void pruneAndArchiveMultipleLogs() throws IOException
assertEquals( 6, storeDir.listFiles( LogFiles.FILENAME_FILTER ).length ); assertEquals( 6, storeDir.listFiles( LogFiles.FILENAME_FILTER ).length );
assertEquals( byteOffset, highestCorrectLogFile.length() ); assertEquals( byteOffset, highestCorrectLogFile.length() );


File corruptedLogsDirectory = new File( storeDir, CorruptedLogsTruncator.CORRUPTED_TX_LOGS_FOLDER_NAME ); File corruptedLogsDirectory = new File( storeDir, CorruptedLogsTruncator.CORRUPTED_TX_LOGS_BASE_NAME );
assertTrue( corruptedLogsDirectory.exists() ); assertTrue( corruptedLogsDirectory.exists() );
File[] files = corruptedLogsDirectory.listFiles(); File[] files = corruptedLogsDirectory.listFiles();
assertEquals( 1, files.length ); assertEquals( 1, files.length );
Expand Down Expand Up @@ -176,7 +176,7 @@ private void checkEntryNameAndSize( ZipFile zipFile, String entryName, int expec
private void checkArchiveName( long highestLogVersion, long byteOffset, File corruptedLogsArchive ) private void checkArchiveName( long highestLogVersion, long byteOffset, File corruptedLogsArchive )
{ {
String name = corruptedLogsArchive.getName(); String name = corruptedLogsArchive.getName();
assertTrue( name.startsWith( "corrupted-logs-" + highestLogVersion + "-" + byteOffset ) ); assertTrue( name.startsWith( "corrupted-neostore.transaction.db-" + highestLogVersion + "-" + byteOffset ) );
assertTrue( FilenameUtils.isExtension( name, "zip" ) ); assertTrue( FilenameUtils.isExtension( name, "zip" ) );
} }


Expand Down
Expand Up @@ -608,9 +608,7 @@ private static class FirstTxIdConfigurableTailScanner extends LogTailScanner
@Override @Override
protected ExtractedTransactionRecord extractFirstTxIdAfterPosition( LogPosition initialPosition, long maxLogVersion ) protected ExtractedTransactionRecord extractFirstTxIdAfterPosition( LogPosition initialPosition, long maxLogVersion )
{ {
ExtractedTransactionRecord record = new ExtractedTransactionRecord(); return new ExtractedTransactionRecord( txId );
record.setId( txId );
return record;
} }
} }
} }

0 comments on commit a4307ba

Please sign in to comment.