Skip to content

Commit

Permalink
Renaming and javadocing.
Browse files Browse the repository at this point in the history
  • Loading branch information
MishaDemianenko committed Oct 3, 2017
1 parent 0013888 commit 228b054
Show file tree
Hide file tree
Showing 8 changed files with 41 additions and 29 deletions.
Expand Up @@ -150,14 +150,14 @@
import org.neo4j.kernel.lifecycle.Lifecycles; import org.neo4j.kernel.lifecycle.Lifecycles;
import org.neo4j.kernel.monitoring.Monitors; import org.neo4j.kernel.monitoring.Monitors;
import org.neo4j.kernel.monitoring.tracing.Tracers; import org.neo4j.kernel.monitoring.tracing.Tracers;
import org.neo4j.kernel.recovery.CorruptedLogsTruncator;
import org.neo4j.kernel.recovery.DefaultRecoveryService; import org.neo4j.kernel.recovery.DefaultRecoveryService;
import org.neo4j.kernel.recovery.LogTailScanner; import org.neo4j.kernel.recovery.LogTailScanner;
import org.neo4j.kernel.recovery.LoggingLogTailScannerMonitor; import org.neo4j.kernel.recovery.LoggingLogTailScannerMonitor;
import org.neo4j.kernel.recovery.PositionToRecoverFrom; import org.neo4j.kernel.recovery.PositionToRecoverFrom;
import org.neo4j.kernel.recovery.Recovery; import org.neo4j.kernel.recovery.Recovery;
import org.neo4j.kernel.recovery.RecoveryMonitor; import org.neo4j.kernel.recovery.RecoveryMonitor;
import org.neo4j.kernel.recovery.RecoveryService; import org.neo4j.kernel.recovery.RecoveryService;
import org.neo4j.kernel.recovery.TransactionLogPruner;
import org.neo4j.kernel.spi.explicitindex.IndexImplementation; import org.neo4j.kernel.spi.explicitindex.IndexImplementation;
import org.neo4j.kernel.spi.explicitindex.IndexProviders; import org.neo4j.kernel.spi.explicitindex.IndexProviders;
import org.neo4j.logging.Log; import org.neo4j.logging.Log;
Expand Down Expand Up @@ -703,8 +703,8 @@ private void buildRecovery(
{ {
RecoveryService recoveryService = new DefaultRecoveryService( storageEngine, tailScanner, transactionIdStore, RecoveryService recoveryService = new DefaultRecoveryService( storageEngine, tailScanner, transactionIdStore,
logicalTransactionStore, logVersionRepository, positionMonitor ); logicalTransactionStore, logVersionRepository, positionMonitor );
TransactionLogPruner logPruner = new TransactionLogPruner( storeDir, logFiles, fileSystemAbstraction ); CorruptedLogsTruncator logsTruncator = new CorruptedLogsTruncator( storeDir, logFiles, fileSystemAbstraction );
Recovery recovery = new Recovery( recoveryService, startupStatistics, logPruner, recoveryMonitor, failOnCorruptedLogFiles ); Recovery recovery = new Recovery( recoveryService, startupStatistics, logsTruncator, recoveryMonitor, failOnCorruptedLogFiles );
life.add( recovery ); life.add( recovery );
} }


Expand Down
Expand Up @@ -27,12 +27,24 @@ public class SimpleLogService extends AbstractLogService
private final LogProvider userLogProvider; private final LogProvider userLogProvider;
private final LogProvider internalLogProvider; private final LogProvider internalLogProvider;


/**
* Create log service where both: user and internal log provider use the same {@link LogProvider} as a provider.
* Should be used when user and internal are backed by same log provider.
* @param commonLogProvider log provider
*/
public SimpleLogService( LogProvider commonLogProvider ) public SimpleLogService( LogProvider commonLogProvider )
{ {
this.userLogProvider = commonLogProvider; this.userLogProvider = commonLogProvider;
this.internalLogProvider = commonLogProvider; this.internalLogProvider = commonLogProvider;
} }


/**
* Create log service with different user and internal log providers.
* User logs will be duplicated to internal logs as well.
* Should be used when user and internal are backed by different log providers.
* @param userLogProvider user log provider
* @param internalLogProvider internal log provider
*/
public SimpleLogService( LogProvider userLogProvider, LogProvider internalLogProvider ) public SimpleLogService( LogProvider userLogProvider, LogProvider internalLogProvider )
{ {
this.userLogProvider = new DuplicatingLogProvider( userLogProvider, internalLogProvider ); this.userLogProvider = new DuplicatingLogProvider( userLogProvider, internalLogProvider );
Expand Down
Expand Up @@ -37,14 +37,14 @@
import static java.lang.String.format; import static java.lang.String.format;


/** /**
* Transaction log pruner used during recovery to truncate all the logs after some specified position, that * Transaction log truncator used during recovery to truncate all the logs after some specified position, that
* recovery threats as corrupted or non-readable. * recovery threats as corrupted or non-readable.
* Transaction log file specified by provided log position will be truncated to provided length, any * Transaction log file specified by provided log position will be truncated to provided length, any
* subsequent files will be removed. * subsequent files will be removed.
* Any removed or modified log content will be stored in separate corruption logs archive for further analysis and as * Any removed or modified log content will be stored in separate corruption logs archive for further analysis and as
* an additional safety option to have the possibility to fully restore original logs in a faulty case. * an additional safety option to have the possibility to fully restore original logs in a faulty case.
*/ */
public class TransactionLogPruner public class CorruptedLogsTruncator
{ {
public static final String CORRUPTED_TX_LOGS_FOLDER_NAME = "corrupted-tx-logs"; public static final String CORRUPTED_TX_LOGS_FOLDER_NAME = "corrupted-tx-logs";
private static final String LOG_FILE_ARCHIVE_PATTERN = "corrupted-logs-%d-%d-%d.zip"; private static final String LOG_FILE_ARCHIVE_PATTERN = "corrupted-logs-%d-%d-%d.zip";
Expand All @@ -53,21 +53,21 @@ public class TransactionLogPruner
private final PhysicalLogFiles logFiles; private final PhysicalLogFiles logFiles;
private final FileSystemAbstraction fs; private final FileSystemAbstraction fs;


public TransactionLogPruner( File storeDir, PhysicalLogFiles logFiles, FileSystemAbstraction fs ) public CorruptedLogsTruncator( File storeDir, PhysicalLogFiles logFiles, FileSystemAbstraction fs )
{ {
this.storeDir = storeDir; this.storeDir = storeDir;
this.logFiles = logFiles; this.logFiles = logFiles;
this.fs = fs; this.fs = fs;
} }


/** /**
* Prune all transaction logs after provided position. Log version specified in a position will be * Truncate all transaction logs after provided position. Log version specified in a position will be
* truncated to provided byte offset, any subsequent log files will be deleted. Backup copy of removed data will * truncated to provided byte offset, any subsequent log files will be deleted. Backup copy of removed data will
* be stored in separate archive. * be stored in separate archive.
* @param positionAfterLastRecoveredTransaction position after last recovered transaction * @param positionAfterLastRecoveredTransaction position after last recovered transaction
* @throws IOException * @throws IOException
*/ */
public void prune( LogPosition positionAfterLastRecoveredTransaction ) throws IOException public void truncate( LogPosition positionAfterLastRecoveredTransaction ) throws IOException
{ {
long recoveredTransactionLogVersion = positionAfterLastRecoveredTransaction.getLogVersion(); long recoveredTransactionLogVersion = positionAfterLastRecoveredTransaction.getLogVersion();
long recoveredTransactionOffset = positionAfterLastRecoveredTransaction.getByteOffset(); long recoveredTransactionOffset = positionAfterLastRecoveredTransaction.getByteOffset();
Expand Down
Expand Up @@ -41,17 +41,17 @@ public class Recovery extends LifecycleAdapter
private final RecoveryService recoveryService; private final RecoveryService recoveryService;
private final RecoveryMonitor monitor; private final RecoveryMonitor monitor;
private final StartupStatisticsProvider startupStatistics; private final StartupStatisticsProvider startupStatistics;
private final TransactionLogPruner logPruner; private final CorruptedLogsTruncator logsTruncator;
private final boolean failOnCorruptedLogFiles; private final boolean failOnCorruptedLogFiles;
private int numberOfRecoveredTransactions; private int numberOfRecoveredTransactions;


public Recovery( RecoveryService recoveryService, StartupStatisticsProvider startupStatistics, public Recovery( RecoveryService recoveryService, StartupStatisticsProvider startupStatistics,
TransactionLogPruner logPruner, RecoveryMonitor monitor, boolean failOnCorruptedLogFiles ) CorruptedLogsTruncator logsTruncator, RecoveryMonitor monitor, boolean failOnCorruptedLogFiles )
{ {
this.recoveryService = recoveryService; this.recoveryService = recoveryService;
this.monitor = monitor; this.monitor = monitor;
this.startupStatistics = startupStatistics; this.startupStatistics = startupStatistics;
this.logPruner = logPruner; this.logsTruncator = logsTruncator;
this.failOnCorruptedLogFiles = failOnCorruptedLogFiles; this.failOnCorruptedLogFiles = failOnCorruptedLogFiles;
} }


Expand Down Expand Up @@ -121,7 +121,7 @@ public void init() throws Throwable
recoveryToPosition = recoveryFromPosition; recoveryToPosition = recoveryFromPosition;
} }
} }
logPruner.prune( recoveryToPosition ); logsTruncator.truncate( recoveryToPosition );


recoveryService.transactionsRecovered( lastTransaction, recoveryToPosition ); recoveryService.transactionsRecovered( lastTransaction, recoveryToPosition );
startupStatistics.setNumberOfRecoveredTransactions( numberOfRecoveredTransactions ); startupStatistics.setNumberOfRecoveredTransactions( numberOfRecoveredTransactions );
Expand Down
Expand Up @@ -67,7 +67,7 @@
import org.neo4j.kernel.internal.GraphDatabaseAPI; import org.neo4j.kernel.internal.GraphDatabaseAPI;
import org.neo4j.kernel.lifecycle.Lifespan; import org.neo4j.kernel.lifecycle.Lifespan;
import org.neo4j.kernel.monitoring.Monitors; import org.neo4j.kernel.monitoring.Monitors;
import org.neo4j.kernel.recovery.TransactionLogPruner; import org.neo4j.kernel.recovery.CorruptedLogsTruncator;
import org.neo4j.logging.AssertableLogProvider; import org.neo4j.logging.AssertableLogProvider;
import org.neo4j.storageengine.api.StorageCommand; import org.neo4j.storageengine.api.StorageCommand;
import org.neo4j.test.TestGraphDatabaseFactory; import org.neo4j.test.TestGraphDatabaseFactory;
Expand Down Expand Up @@ -397,7 +397,7 @@ public void repetitiveRecoveryIfCorruptedLogsWithCheckpoints() throws IOExceptio
assertThat( numberOfRecoveredTransactions, Matchers.greaterThanOrEqualTo( 0 ) ); assertThat( numberOfRecoveredTransactions, Matchers.greaterThanOrEqualTo( 0 ) );
} }


File corruptedLogArchives = new File( storeDir, TransactionLogPruner.CORRUPTED_TX_LOGS_FOLDER_NAME ); File corruptedLogArchives = new File( storeDir, CorruptedLogsTruncator.CORRUPTED_TX_LOGS_FOLDER_NAME );
assertThat( corruptedLogArchives.listFiles(), not( emptyArray() ) ); assertThat( corruptedLogArchives.listFiles(), not( emptyArray() ) );
} }


Expand Down
Expand Up @@ -58,12 +58,12 @@
import org.neo4j.kernel.impl.transaction.log.entry.VersionAwareLogEntryReader; import org.neo4j.kernel.impl.transaction.log.entry.VersionAwareLogEntryReader;
import org.neo4j.kernel.lifecycle.LifeSupport; import org.neo4j.kernel.lifecycle.LifeSupport;
import org.neo4j.kernel.monitoring.Monitors; import org.neo4j.kernel.monitoring.Monitors;
import org.neo4j.kernel.recovery.CorruptedLogsTruncator;
import org.neo4j.kernel.recovery.DefaultRecoveryService; import org.neo4j.kernel.recovery.DefaultRecoveryService;
import org.neo4j.kernel.recovery.LogTailScanner; import org.neo4j.kernel.recovery.LogTailScanner;
import org.neo4j.kernel.recovery.Recovery; import org.neo4j.kernel.recovery.Recovery;
import org.neo4j.kernel.recovery.RecoveryApplier; import org.neo4j.kernel.recovery.RecoveryApplier;
import org.neo4j.kernel.recovery.RecoveryMonitor; import org.neo4j.kernel.recovery.RecoveryMonitor;
import org.neo4j.kernel.recovery.TransactionLogPruner;
import org.neo4j.logging.AssertableLogProvider; import org.neo4j.logging.AssertableLogProvider;
import org.neo4j.storageengine.api.StorageEngine; import org.neo4j.storageengine.api.StorageEngine;
import org.neo4j.storageengine.api.TransactionApplicationMode; import org.neo4j.storageengine.api.TransactionApplicationMode;
Expand Down Expand Up @@ -157,7 +157,7 @@ public void shouldRecoverExistingData() throws Exception
mock( PhysicalLogFile.Monitor.class ), logHeaderCache ) ); mock( PhysicalLogFile.Monitor.class ), logHeaderCache ) );
LogicalTransactionStore txStore = new PhysicalLogicalTransactionStore( logFile, metadataCache, reader, LogicalTransactionStore txStore = new PhysicalLogicalTransactionStore( logFile, metadataCache, reader,
monitors, false ); monitors, false );
TransactionLogPruner logPruner = new TransactionLogPruner( storeDir, logFiles, fileSystemRule.get() ); CorruptedLogsTruncator logPruner = new CorruptedLogsTruncator( storeDir, logFiles, fileSystemRule.get() );
life.add( new Recovery( new DefaultRecoveryService( storageEngine, tailScanner, transactionIdStore, life.add( new Recovery( new DefaultRecoveryService( storageEngine, tailScanner, transactionIdStore,
txStore, versionRepository, NO_MONITOR ) txStore, versionRepository, NO_MONITOR )
{ {
Expand Down Expand Up @@ -262,7 +262,7 @@ public void shouldSeeThatACleanDatabaseShouldNotRequireRecovery() throws Excepti
mock( PhysicalLogFile.Monitor.class ), logHeaderCache ) ); mock( PhysicalLogFile.Monitor.class ), logHeaderCache ) );
LogicalTransactionStore txStore = new PhysicalLogicalTransactionStore( logFile, metadataCache, reader, monitors, LogicalTransactionStore txStore = new PhysicalLogicalTransactionStore( logFile, metadataCache, reader, monitors,
false ); false );
TransactionLogPruner logPruner = new TransactionLogPruner( storeDir, logFiles, fileSystemRule.get() ); CorruptedLogsTruncator logPruner = new CorruptedLogsTruncator( storeDir, logFiles, fileSystemRule.get() );
life.add( new Recovery( new DefaultRecoveryService( storageEngine, tailScanner, transactionIdStore, life.add( new Recovery( new DefaultRecoveryService( storageEngine, tailScanner, transactionIdStore,
txStore, versionRepository, NO_MONITOR ) txStore, versionRepository, NO_MONITOR )
{ {
Expand Down Expand Up @@ -404,7 +404,7 @@ private boolean recover( File storeDir, PhysicalLogFiles logFiles )
mock( PhysicalLogFile.Monitor.class ), logHeaderCache ) ); mock( PhysicalLogFile.Monitor.class ), logHeaderCache ) );
LogicalTransactionStore txStore = new PhysicalLogicalTransactionStore( logFile, metadataCache, reader, monitors, LogicalTransactionStore txStore = new PhysicalLogicalTransactionStore( logFile, metadataCache, reader, monitors,
false ); false );
TransactionLogPruner logPruner = new TransactionLogPruner( storeDir, logFiles, fileSystemRule.get() ); CorruptedLogsTruncator logPruner = new CorruptedLogsTruncator( storeDir, logFiles, fileSystemRule.get() );
life.add( new Recovery( new DefaultRecoveryService( storageEngine, tailScanner, transactionIdStore, life.add( new Recovery( new DefaultRecoveryService( storageEngine, tailScanner, transactionIdStore,
txStore, versionRepository, NO_MONITOR ) txStore, versionRepository, NO_MONITOR )
{ {
Expand Down
Expand Up @@ -43,11 +43,11 @@
import org.neo4j.kernel.internal.DatabaseHealth; import org.neo4j.kernel.internal.DatabaseHealth;
import org.neo4j.kernel.lifecycle.LifeSupport; import org.neo4j.kernel.lifecycle.LifeSupport;
import org.neo4j.kernel.monitoring.Monitors; import org.neo4j.kernel.monitoring.Monitors;
import org.neo4j.kernel.recovery.CorruptedLogsTruncator;
import org.neo4j.kernel.recovery.Recovery; import org.neo4j.kernel.recovery.Recovery;
import org.neo4j.kernel.recovery.RecoveryApplier; import org.neo4j.kernel.recovery.RecoveryApplier;
import org.neo4j.kernel.recovery.RecoveryMonitor; import org.neo4j.kernel.recovery.RecoveryMonitor;
import org.neo4j.kernel.recovery.RecoveryService; import org.neo4j.kernel.recovery.RecoveryService;
import org.neo4j.kernel.recovery.TransactionLogPruner;
import org.neo4j.storageengine.api.StorageCommand; import org.neo4j.storageengine.api.StorageCommand;
import org.neo4j.storageengine.api.TransactionApplicationMode; import org.neo4j.storageengine.api.TransactionApplicationMode;
import org.neo4j.test.rule.TestDirectory; import org.neo4j.test.rule.TestDirectory;
Expand Down Expand Up @@ -197,7 +197,7 @@ public void shouldOpenAndRecoverExistingData() throws Exception


life.add( new BatchingTransactionAppender( logFile, NO_ROTATION, positionCache, life.add( new BatchingTransactionAppender( logFile, NO_ROTATION, positionCache,
transactionIdStore, BYPASS, DATABASE_HEALTH ) ); transactionIdStore, BYPASS, DATABASE_HEALTH ) );
TransactionLogPruner logPruner = new TransactionLogPruner( testDir, logFiles, fileSystemRule.get() ); CorruptedLogsTruncator logPruner = new CorruptedLogsTruncator( testDir, logFiles, fileSystemRule.get() );
life.add( new Recovery( new RecoveryService() life.add( new Recovery( new RecoveryService()
{ {
@Override @Override
Expand Down
Expand Up @@ -48,7 +48,7 @@
import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue; import static org.junit.Assert.assertTrue;


public class TransactionLogPrunerTest public class CorruptedLogsTruncatorTest
{ {
private static final int SINGLE_LOG_FILE_SIZE = 25; private static final int SINGLE_LOG_FILE_SIZE = 25;
private static final int TOTAL_NUMBER_OF_LOG_FILES = 11; private static final int TOTAL_NUMBER_OF_LOG_FILES = 11;
Expand All @@ -57,20 +57,20 @@ public class TransactionLogPrunerTest
public FileSystemRule fileSystemRule = new DefaultFileSystemRule(); public FileSystemRule fileSystemRule = new DefaultFileSystemRule();
private File storeDir; private File storeDir;
private PhysicalLogFiles logFiles; private PhysicalLogFiles logFiles;
private TransactionLogPruner logPruner; private CorruptedLogsTruncator logPruner;


@Before @Before
public void setUp() throws Exception public void setUp() throws Exception
{ {
storeDir = testDirectory.graphDbDir(); storeDir = testDirectory.graphDbDir();
logFiles = new PhysicalLogFiles( storeDir, fileSystemRule ); logFiles = new PhysicalLogFiles( storeDir, fileSystemRule );
logPruner = new TransactionLogPruner( storeDir, logFiles, fileSystemRule ); logPruner = new CorruptedLogsTruncator( storeDir, logFiles, fileSystemRule );
} }


@Test @Test
public void doNotPruneEmptyLogs() throws IOException public void doNotPruneEmptyLogs() throws IOException
{ {
logPruner.prune( LogPosition.start( 0 ) ); logPruner.truncate( LogPosition.start( 0 ) );
assertTrue( FileUtils.isEmptyDirectory( storeDir ) ); assertTrue( FileUtils.isEmptyDirectory( storeDir ) );
} }


Expand All @@ -84,7 +84,7 @@ public void doNotPruneNonCorruptedLogs() throws IOException
LogPosition endOfLogsPosition = new LogPosition( highestLogVersion, fileSizeBeforePrune ); LogPosition endOfLogsPosition = new LogPosition( highestLogVersion, fileSizeBeforePrune );
assertEquals( TOTAL_NUMBER_OF_LOG_FILES - 1, highestLogVersion ); assertEquals( TOTAL_NUMBER_OF_LOG_FILES - 1, highestLogVersion );


logPruner.prune( endOfLogsPosition ); logPruner.truncate( endOfLogsPosition );


assertEquals( TOTAL_NUMBER_OF_LOG_FILES, storeDir.listFiles( LogFiles.FILENAME_FILTER ).length ); assertEquals( TOTAL_NUMBER_OF_LOG_FILES, storeDir.listFiles( LogFiles.FILENAME_FILTER ).length );
assertEquals( fileSizeBeforePrune, logFiles.getHighestLogFile().length() ); assertEquals( fileSizeBeforePrune, logFiles.getHighestLogFile().length() );
Expand All @@ -103,12 +103,12 @@ public void pruneAndArchiveLastLog() throws IOException
long byteOffset = fileSizeBeforePrune - bytesToPrune; long byteOffset = fileSizeBeforePrune - bytesToPrune;
LogPosition prunePosition = new LogPosition( highestLogVersion, byteOffset ); LogPosition prunePosition = new LogPosition( highestLogVersion, byteOffset );


logPruner.prune( prunePosition ); logPruner.truncate( prunePosition );


assertEquals( TOTAL_NUMBER_OF_LOG_FILES, storeDir.listFiles( LogFiles.FILENAME_FILTER ).length ); assertEquals( TOTAL_NUMBER_OF_LOG_FILES, storeDir.listFiles( LogFiles.FILENAME_FILTER ).length );
assertEquals( byteOffset, highestLogFile.length() ); assertEquals( byteOffset, highestLogFile.length() );


File corruptedLogsDirectory = new File( storeDir, TransactionLogPruner.CORRUPTED_TX_LOGS_FOLDER_NAME ); File corruptedLogsDirectory = new File( storeDir, CorruptedLogsTruncator.CORRUPTED_TX_LOGS_FOLDER_NAME );
assertTrue( corruptedLogsDirectory.exists() ); assertTrue( corruptedLogsDirectory.exists() );
File[] files = corruptedLogsDirectory.listFiles(); File[] files = corruptedLogsDirectory.listFiles();
assertEquals( 1, files.length ); assertEquals( 1, files.length );
Expand All @@ -134,12 +134,12 @@ public void pruneAndArchiveMultipleLogs() throws IOException
long byteOffset = fileSizeBeforePrune - bytesToPrune; long byteOffset = fileSizeBeforePrune - bytesToPrune;
LogPosition prunePosition = new LogPosition( highestCorrectLogFileIndex, byteOffset ); LogPosition prunePosition = new LogPosition( highestCorrectLogFileIndex, byteOffset );


logPruner.prune( prunePosition ); logPruner.truncate( prunePosition );


assertEquals( 6, storeDir.listFiles( LogFiles.FILENAME_FILTER ).length ); assertEquals( 6, storeDir.listFiles( LogFiles.FILENAME_FILTER ).length );
assertEquals( byteOffset, highestCorrectLogFile.length() ); assertEquals( byteOffset, highestCorrectLogFile.length() );


File corruptedLogsDirectory = new File( storeDir, TransactionLogPruner.CORRUPTED_TX_LOGS_FOLDER_NAME ); File corruptedLogsDirectory = new File( storeDir, CorruptedLogsTruncator.CORRUPTED_TX_LOGS_FOLDER_NAME );
assertTrue( corruptedLogsDirectory.exists() ); assertTrue( corruptedLogsDirectory.exists() );
File[] files = corruptedLogsDirectory.listFiles(); File[] files = corruptedLogsDirectory.listFiles();
assertEquals( 1, files.length ); assertEquals( 1, files.length );
Expand Down

0 comments on commit 228b054

Please sign in to comment.