Skip to content

Commit

Permalink
Address PR comments
Browse files Browse the repository at this point in the history
  • Loading branch information
MishaDemianenko committed Oct 3, 2017
1 parent 228b054 commit a4307ba
Show file tree
Hide file tree
Showing 5 changed files with 32 additions and 31 deletions.
Expand Up @@ -32,22 +32,23 @@
import org.neo4j.io.fs.FileSystemAbstraction;
import org.neo4j.io.fs.StoreChannel;
import org.neo4j.kernel.impl.transaction.log.LogPosition;
import org.neo4j.kernel.impl.transaction.log.PhysicalLogFile;
import org.neo4j.kernel.impl.transaction.log.PhysicalLogFiles;

import static java.lang.String.format;

/**
* Transaction log truncator used during recovery to truncate all the logs after some specified position, that
* recovery threats as corrupted or non-readable.
* recovery treats as corrupted or non-readable.
* Transaction log file specified by provided log position will be truncated to provided length, any
* subsequent files will be removed.
* Any removed or modified log content will be stored in separate corruption logs archive for further analysis and as
* an additional safety option to have the possibility to fully restore original logs in a faulty case.
*/
public class CorruptedLogsTruncator
{
public static final String CORRUPTED_TX_LOGS_FOLDER_NAME = "corrupted-tx-logs";
private static final String LOG_FILE_ARCHIVE_PATTERN = "corrupted-logs-%d-%d-%d.zip";
public static final String CORRUPTED_TX_LOGS_BASE_NAME = "corrupted-" + PhysicalLogFile.DEFAULT_NAME;
private static final String LOG_FILE_ARCHIVE_PATTERN = CORRUPTED_TX_LOGS_BASE_NAME + "-%d-%d-%d.zip";

private final File storeDir;
private final PhysicalLogFiles logFiles;
Expand All @@ -72,7 +73,7 @@ public void truncate( LogPosition positionAfterLastRecoveredTransaction ) throws
long recoveredTransactionLogVersion = positionAfterLastRecoveredTransaction.getLogVersion();
long recoveredTransactionOffset = positionAfterLastRecoveredTransaction.getByteOffset();
if ( isRecoveredLogCorrupted( recoveredTransactionLogVersion, recoveredTransactionOffset ) ||
isNotLastLogFile( recoveredTransactionLogVersion ) )
haveMoreRecentLogFiles( recoveredTransactionLogVersion ) )
{
backupCorruptedContent( recoveredTransactionLogVersion, recoveredTransactionOffset );
truncateLogFiles( recoveredTransactionLogVersion, recoveredTransactionOffset );
Expand Down Expand Up @@ -124,7 +125,7 @@ private void backupCorruptedContent( long recoveredTransactionLogVersion, long r
private File getArchiveFile( long recoveredTransactionLogVersion, long recoveredTransactionOffset )
throws IOException
{
File corruptedLogsFolder = new File( storeDir, CORRUPTED_TX_LOGS_FOLDER_NAME );
File corruptedLogsFolder = new File( storeDir, CORRUPTED_TX_LOGS_BASE_NAME );
fs.mkdirs( corruptedLogsFolder );
return new File( corruptedLogsFolder,
format( LOG_FILE_ARCHIVE_PATTERN, recoveredTransactionLogVersion, recoveredTransactionOffset,
Expand Down Expand Up @@ -155,7 +156,7 @@ private void copyTransactionLogContent( long logFileIndex, long logOffset, ZipOu
destination.closeEntry();
}

private boolean isNotLastLogFile( long recoveredTransactionLogVersion )
private boolean haveMoreRecentLogFiles( long recoveredTransactionLogVersion )
{
return logFiles.getHighestLogVersion() > recoveredTransactionLogVersion;
}
Expand Down
Expand Up @@ -189,7 +189,6 @@ protected LogTailInformation checkpointTailInformation( long highestLogVersion,
protected ExtractedTransactionRecord extractFirstTxIdAfterPosition( LogPosition initialPosition, long maxLogVersion ) throws IOException
{
LogPosition currentPosition = initialPosition;
ExtractedTransactionRecord transactionRecord = new ExtractedTransactionRecord();
while ( currentPosition.getLogVersion() <= maxLogVersion )
{
LogVersionedStoreChannel storeChannel = PhysicalLogFile.tryOpenForVersion( logFiles, fileSystem,
Expand All @@ -207,17 +206,15 @@ protected ExtractedTransactionRecord extractFirstTxIdAfterPosition( LogPosition
LogEntry entry = cursor.get();
if ( entry instanceof LogEntryCommit )
{
transactionRecord.setId( ((LogEntryCommit) entry).getTxId() );
return transactionRecord;
return new ExtractedTransactionRecord( ((LogEntryCommit) entry).getTxId() );
}
}
}
}
catch ( Throwable t )
{
monitor.corruptedLogFile( currentPosition.getLogVersion(), t );
transactionRecord.setFailure( true );
return transactionRecord;
return new ExtractedTransactionRecord( true );
}
finally
{
Expand All @@ -227,7 +224,7 @@ protected ExtractedTransactionRecord extractFirstTxIdAfterPosition( LogPosition

currentPosition = LogPosition.start( currentPosition.getLogVersion() + 1 );
}
return transactionRecord;
return new ExtractedTransactionRecord();
}

/**
Expand Down Expand Up @@ -260,33 +257,38 @@ public LogTailInformation getTailInformation() throws UnderlyingStorageException

static class ExtractedTransactionRecord
{
private long id;
private boolean failure;
private final long id;
private final boolean failure;

ExtractedTransactionRecord()
{
id = NO_TRANSACTION_ID;
failure = false;
this( NO_TRANSACTION_ID, false );
}

public long getId()
ExtractedTransactionRecord( long txId )
{
return id;
this( txId, false );
}

public void setId( long id )
ExtractedTransactionRecord( boolean failure )
{
this.id = id;
this( NO_TRANSACTION_ID, failure );
}

public boolean isFailure()
private ExtractedTransactionRecord( long txId, boolean failure )
{
return failure;
this.id = txId;
this.failure = failure;
}

public void setFailure( boolean failure )
public long getId()
{
this.failure = failure;
return id;
}

public boolean isFailure()
{
return failure;
}
}

Expand Down
Expand Up @@ -397,7 +397,7 @@ public void repetitiveRecoveryIfCorruptedLogsWithCheckpoints() throws IOExceptio
assertThat( numberOfRecoveredTransactions, Matchers.greaterThanOrEqualTo( 0 ) );
}

File corruptedLogArchives = new File( storeDir, CorruptedLogsTruncator.CORRUPTED_TX_LOGS_FOLDER_NAME );
File corruptedLogArchives = new File( storeDir, CorruptedLogsTruncator.CORRUPTED_TX_LOGS_BASE_NAME );
assertThat( corruptedLogArchives.listFiles(), not( emptyArray() ) );
}

Expand Down
Expand Up @@ -108,7 +108,7 @@ public void pruneAndArchiveLastLog() throws IOException
assertEquals( TOTAL_NUMBER_OF_LOG_FILES, storeDir.listFiles( LogFiles.FILENAME_FILTER ).length );
assertEquals( byteOffset, highestLogFile.length() );

File corruptedLogsDirectory = new File( storeDir, CorruptedLogsTruncator.CORRUPTED_TX_LOGS_FOLDER_NAME );
File corruptedLogsDirectory = new File( storeDir, CorruptedLogsTruncator.CORRUPTED_TX_LOGS_BASE_NAME );
assertTrue( corruptedLogsDirectory.exists() );
File[] files = corruptedLogsDirectory.listFiles();
assertEquals( 1, files.length );
Expand Down Expand Up @@ -139,7 +139,7 @@ public void pruneAndArchiveMultipleLogs() throws IOException
assertEquals( 6, storeDir.listFiles( LogFiles.FILENAME_FILTER ).length );
assertEquals( byteOffset, highestCorrectLogFile.length() );

File corruptedLogsDirectory = new File( storeDir, CorruptedLogsTruncator.CORRUPTED_TX_LOGS_FOLDER_NAME );
File corruptedLogsDirectory = new File( storeDir, CorruptedLogsTruncator.CORRUPTED_TX_LOGS_BASE_NAME );
assertTrue( corruptedLogsDirectory.exists() );
File[] files = corruptedLogsDirectory.listFiles();
assertEquals( 1, files.length );
Expand Down Expand Up @@ -176,7 +176,7 @@ private void checkEntryNameAndSize( ZipFile zipFile, String entryName, int expec
private void checkArchiveName( long highestLogVersion, long byteOffset, File corruptedLogsArchive )
{
String name = corruptedLogsArchive.getName();
assertTrue( name.startsWith( "corrupted-logs-" + highestLogVersion + "-" + byteOffset ) );
assertTrue( name.startsWith( "corrupted-neostore.transaction.db-" + highestLogVersion + "-" + byteOffset ) );
assertTrue( FilenameUtils.isExtension( name, "zip" ) );
}

Expand Down
Expand Up @@ -608,9 +608,7 @@ private static class FirstTxIdConfigurableTailScanner extends LogTailScanner
@Override
protected ExtractedTransactionRecord extractFirstTxIdAfterPosition( LogPosition initialPosition, long maxLogVersion )
{
ExtractedTransactionRecord record = new ExtractedTransactionRecord();
record.setId( txId );
return record;
return new ExtractedTransactionRecord( txId );
}
}
}

0 comments on commit a4307ba

Please sign in to comment.