From 76978820ea233d2e11d35eccfba46f3f9d3e26b4 Mon Sep 17 00:00:00 2001 From: Anton Klaren Date: Thu, 24 Aug 2017 14:09:42 +0200 Subject: [PATCH] LogTailScanner updated LogTailScanner is now invoked once during startup to collect information about the tail of the transaction logs, and this result it reused by all component interested in such information. If any log entries with unsupported version is found, a better error message is thrown to avoid confusion as to what went wrong. We now also properly respect dbms.allow_upgrade when the transaction log format is changed. --- .../consistency/ConsistencyCheckToolTest.java | 2 +- .../org/neo4j/kernel/NeoStoreDataSource.java | 40 +++-- .../kernel/NeoStoreTransactionLogModule.java | 5 - .../recovery/RecoveryRequiredChecker.java | 5 +- .../recordstorage/RecordStorageEngine.java | 9 -- .../impl/storemigration/DatabaseMigrator.java | 8 +- .../storemigration/UpgradableDatabase.java | 27 +--- .../impl/transaction/log/LogTailScanner.java | 92 +++++------ .../log/LogVersionUpgradeChecker.java | 54 +++++++ .../log/entry/LogEntryVersion.java | 38 ++++- .../transaction/log/entry/LogEntryWriter.java | 19 ++- .../kernel/recovery/DefaultRecoverySPI.java | 7 +- .../recovery/PositionToRecoverFrom.java | 11 +- .../storageengine/api/StorageEngine.java | 6 - .../java/org/neo4j/kernel/RecoveryTest.java | 18 +-- .../kernel/impl/store/TestStoreAccess.java | 2 +- .../storemigration/MigrationTestUtils.java | 4 +- .../participant/StoreMigratorIT.java | 32 ++-- .../transaction/log/LogTailScannerTest.java | 67 ++++---- .../log/LogVersionUpgradeCheckerIT.java | 152 ++++++++++++++++++ .../log/LogVersionUpgradeCheckerTest.java | 82 ++++++++++ .../log/entry/LogEntryVersionTest.java | 31 ++++ .../recovery/PositionToRecoverFromTest.java | 27 ++-- .../StoreUpgraderInterruptionTestIT.java | 20 ++- .../test/java/upgrade/StoreUpgraderTest.java | 53 +++--- .../tx/TransactionLogCatchUpWriterTest.java | 7 +- .../impl/storemigration/StoreMigrationIT.java | 7 +- .../UpgradableDatabaseTest.java | 29 ++-- .../neo4j/tools/migration/StoreMigration.java | 44 ++++- 29 files changed, 636 insertions(+), 262 deletions(-) create mode 100644 community/kernel/src/main/java/org/neo4j/kernel/impl/transaction/log/LogVersionUpgradeChecker.java create mode 100644 community/kernel/src/test/java/org/neo4j/kernel/impl/transaction/log/LogVersionUpgradeCheckerIT.java create mode 100644 community/kernel/src/test/java/org/neo4j/kernel/impl/transaction/log/LogVersionUpgradeCheckerTest.java diff --git a/community/consistency-check/src/test/java/org/neo4j/consistency/ConsistencyCheckToolTest.java b/community/consistency-check/src/test/java/org/neo4j/consistency/ConsistencyCheckToolTest.java index 7f406a94c6cff..e2c14b1f012b9 100644 --- a/community/consistency-check/src/test/java/org/neo4j/consistency/ConsistencyCheckToolTest.java +++ b/community/consistency-check/src/test/java/org/neo4j/consistency/ConsistencyCheckToolTest.java @@ -191,7 +191,7 @@ private void createGraphDbAndKillIt() throws Exception tx.success(); } - fs.snapshot( () -> db.shutdown() ); + fs.snapshot( db::shutdown ); } private void runConsistencyCheckToolWith( FileSystemAbstraction fileSystem, String... args ) diff --git a/community/kernel/src/main/java/org/neo4j/kernel/NeoStoreDataSource.java b/community/kernel/src/main/java/org/neo4j/kernel/NeoStoreDataSource.java index e22209313416b..9ff83557614f6 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/NeoStoreDataSource.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/NeoStoreDataSource.java @@ -104,6 +104,7 @@ import org.neo4j.kernel.impl.transaction.log.LogPosition; import org.neo4j.kernel.impl.transaction.log.LogTailScanner; import org.neo4j.kernel.impl.transaction.log.LogVersionRepository; +import org.neo4j.kernel.impl.transaction.log.LogVersionUpgradeChecker; import org.neo4j.kernel.impl.transaction.log.LoggingLogFileMonitor; import org.neo4j.kernel.impl.transaction.log.LogicalTransactionStore; import org.neo4j.kernel.impl.transaction.log.PhysicalLogFile; @@ -164,7 +165,6 @@ import org.neo4j.storageengine.api.StoreReadLayer; import org.neo4j.time.SystemNanoClock; -import static org.neo4j.kernel.impl.transaction.log.entry.InvalidLogEntryHandler.STRICT; import static org.neo4j.kernel.impl.transaction.log.pruning.LogPruneStrategyFactory.fromConfigValue; public class NeoStoreDataSource implements Lifecycle, IndexProviders @@ -427,9 +427,15 @@ public void start() throws IOException life.add( new Delegate( Lifecycles.multiple( indexProviders.values() ) ) ); + // Check the tail of transaction logs and validate version + final PhysicalLogFiles logFiles = new PhysicalLogFiles( storeDir, PhysicalLogFile.DEFAULT_NAME, fs ); + final LogEntryReader logEntryReader = new VersionAwareLogEntryReader<>(); + LogTailScanner tailScanner = new LogTailScanner( logFiles, fs, logEntryReader ); + LogVersionUpgradeChecker.check( tailScanner, config ); + // Upgrade the store before we begin RecordFormats formats = selectStoreFormats( config, storeDir, fs, pageCache, logService ); - upgradeStore( formats ); + upgradeStore( formats, tailScanner ); // Build all modules and their services StorageEngine storageEngine = null; @@ -446,24 +452,21 @@ public void start() throws IOException propertyKeyTokenHolder, labelTokens, relationshipTypeTokens, legacyIndexProviderLookup, indexConfigStore, databaseSchemaState, legacyIndexTransactionOrdering, operationalMode ); - LogEntryReader logEntryReader = - new VersionAwareLogEntryReader<>( storageEngine.commandReaderFactory(), STRICT ); - TransactionIdStore transactionIdStore = dependencies.resolveDependency( TransactionIdStore.class ); LogVersionRepository logVersionRepository = dependencies.resolveDependency( LogVersionRepository.class ); NeoStoreTransactionLogModule transactionLogModule = - buildTransactionLogs( storeDir, config, logProvider, scheduler, fs, + buildTransactionLogs( logFiles, config, logProvider, scheduler, fs, storageEngine, logEntryReader, legacyIndexTransactionOrdering, transactionIdStore, logVersionRepository ); transactionLogModule.satisfyDependencies(dependencies); buildRecovery( fs, transactionIdStore, - logVersionRepository, + tailScanner, monitors.newMonitor( Recovery.Monitor.class ), monitors.newMonitor( PositionToRecoverFrom.Monitor.class ), - transactionLogModule.logFiles(), startupStatistics, - storageEngine, logEntryReader, transactionLogModule.logicalTransactionStore() + logFiles, startupStatistics, + storageEngine, transactionLogModule.logicalTransactionStore() ); // At the time of writing this comes from the storage engine (IndexStoreView) @@ -558,7 +561,7 @@ private static RecordFormats selectStoreFormats( Config config, File storeDir, F return formats; } - private void upgradeStore( RecordFormats format ) + private void upgradeStore( RecordFormats format, LogTailScanner tailScanner ) { VisibleMigrationProgressMonitor progressMonitor = new VisibleMigrationProgressMonitor( logService.getUserLog( StoreMigrator.class ) ); @@ -570,7 +573,7 @@ private void upgradeStore( RecordFormats format ) schemaIndexProviderMap, indexProviders, pageCache, - format ).migrate( storeDir ); + format, tailScanner ).migrate( storeDir ); } private StorageEngine buildStorageEngine( @@ -596,7 +599,7 @@ private StorageEngine buildStorageEngine( } private NeoStoreTransactionLogModule buildTransactionLogs( - File storeDir, + PhysicalLogFiles logFiles, Config config, LogProvider logProvider, JobScheduler scheduler, @@ -607,8 +610,6 @@ private NeoStoreTransactionLogModule buildTransactionLogs( { TransactionMetadataCache transactionMetadataCache = new TransactionMetadataCache( 100_000 ); LogHeaderCache logHeaderCache = new LogHeaderCache( 1000 ); - final PhysicalLogFiles logFiles = new PhysicalLogFiles( storeDir, PhysicalLogFile.DEFAULT_NAME, - fileSystemAbstraction ); final PhysicalLogFile logFile = life.add( new PhysicalLogFile( fileSystemAbstraction, logFiles, config.get( GraphDatabaseSettings.logical_log_rotation_threshold ), @@ -682,20 +683,17 @@ private NeoStoreTransactionLogModule buildTransactionLogs( private void buildRecovery( final FileSystemAbstraction fileSystemAbstraction, TransactionIdStore transactionIdStore, - LogVersionRepository logVersionRepository, + LogTailScanner tailScanner, Recovery.Monitor recoveryMonitor, PositionToRecoverFrom.Monitor positionMonitor, final PhysicalLogFiles logFiles, final StartupStatisticsProvider startupStatistics, StorageEngine storageEngine, - LogEntryReader logEntryReader, LogicalTransactionStore logicalTransactionStore ) { - final LogTailScanner logTailScanner = - new LogTailScanner( logFiles, fileSystemAbstraction, logEntryReader ); - Recovery.SPI spi = new DefaultRecoverySPI( - storageEngine, logFiles, fileSystemAbstraction, logVersionRepository, - logTailScanner, transactionIdStore, logicalTransactionStore, positionMonitor ); + Recovery.SPI spi = + new DefaultRecoverySPI( storageEngine, logFiles, fileSystemAbstraction, tailScanner, transactionIdStore, + logicalTransactionStore, positionMonitor ); Recovery recovery = new Recovery( spi, recoveryMonitor ); monitors.addMonitorListener( new Recovery.Monitor() { diff --git a/community/kernel/src/main/java/org/neo4j/kernel/NeoStoreTransactionLogModule.java b/community/kernel/src/main/java/org/neo4j/kernel/NeoStoreTransactionLogModule.java index e107ffb2d7dd9..74224de6aff10 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/NeoStoreTransactionLogModule.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/NeoStoreTransactionLogModule.java @@ -61,11 +61,6 @@ public LogicalTransactionStore logicalTransactionStore() return logicalTransactionStore; } - public PhysicalLogFiles logFiles() - { - return logFiles; - } - CheckPointer checkPointing() { return checkPointer; diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/recovery/RecoveryRequiredChecker.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/recovery/RecoveryRequiredChecker.java index 5c816c92e6fd9..bc59ecff0adfb 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/recovery/RecoveryRequiredChecker.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/recovery/RecoveryRequiredChecker.java @@ -63,11 +63,12 @@ public boolean isRecoveryRequiredAt( File dataDir ) throws IOException } long logVersion = MetaDataStore.getRecord( pageCache, neoStore, MetaDataStore.Position.LOG_VERSION ); + MetaDataStore.setRecord( pageCache, neoStore, MetaDataStore.Position.LOG_VERSION, logVersion ); PhysicalLogFiles logFiles = new PhysicalLogFiles( dataDir, fs ); LogEntryReader reader = new VersionAwareLogEntryReader<>(); - LogTailScanner finder = new LogTailScanner( logFiles, fs, reader ); - return new PositionToRecoverFrom( finder, NO_MONITOR ).apply( logVersion ) != LogPosition.UNSPECIFIED; + LogTailScanner tailScanner = new LogTailScanner( logFiles, fs, reader ); + return new PositionToRecoverFrom( tailScanner, NO_MONITOR ).get() != LogPosition.UNSPECIFIED; } } diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/storageengine/impl/recordstorage/RecordStorageEngine.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/storageengine/impl/recordstorage/RecordStorageEngine.java index dc79e1685202c..240bbbdae48fc 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/storageengine/impl/recordstorage/RecordStorageEngine.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/storageengine/impl/recordstorage/RecordStorageEngine.java @@ -110,7 +110,6 @@ import org.neo4j.kernel.spi.legacyindex.IndexImplementation; import org.neo4j.logging.LogProvider; import org.neo4j.scheduler.JobScheduler; -import org.neo4j.storageengine.api.CommandReaderFactory; import org.neo4j.storageengine.api.CommandsToApply; import org.neo4j.storageengine.api.StorageCommand; import org.neo4j.storageengine.api.StorageEngine; @@ -153,7 +152,6 @@ public class RecordStorageEngine implements StorageEngine, Lifecycle private final IdOrderingQueue legacyIndexTransactionOrdering; private final LockService lockService; private final WorkSync,LabelUpdateWork> labelScanStoreSync; - private final CommandReaderFactory commandReaderFactory; private final WorkSync indexUpdatesSync; private final IndexStoreView indexStoreView; private final LegacyIndexProviderLookup legacyIndexProviderLookup; @@ -241,7 +239,6 @@ public RecordStorageEngine( labelScanStoreSync = new WorkSync<>( labelScanStore::newWriter ); - commandReaderFactory = new RecordStorageCommandReaderFactory(); indexUpdatesSync = new WorkSync<>( indexingService ); // Immutable state for creating/applying commands @@ -276,12 +273,6 @@ public StoreReadLayer storeReadLayer() return storeLayer; } - @Override - public CommandReaderFactory commandReaderFactory() - { - return commandReaderFactory; - } - @SuppressWarnings( "resource" ) @Override public void createCommands( diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/storemigration/DatabaseMigrator.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/storemigration/DatabaseMigrator.java index 56ddd26161a2b..14ce2d5e65e04 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/storemigration/DatabaseMigrator.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/storemigration/DatabaseMigrator.java @@ -33,6 +33,7 @@ import org.neo4j.kernel.impl.storemigration.participant.LegacyIndexMigrator; import org.neo4j.kernel.impl.storemigration.participant.NativeLabelScanStoreMigrator; import org.neo4j.kernel.impl.storemigration.participant.StoreMigrator; +import org.neo4j.kernel.impl.transaction.log.LogTailScanner; import org.neo4j.kernel.spi.legacyindex.IndexImplementation; import org.neo4j.logging.LogProvider; @@ -53,12 +54,13 @@ public class DatabaseMigrator private final Map indexProviders; private final PageCache pageCache; private final RecordFormats format; + private final LogTailScanner tailScanner; public DatabaseMigrator( MigrationProgressMonitor progressMonitor, FileSystemAbstraction fs, Config config, LogService logService, SchemaIndexProviderMap schemaIndexProviderMap, Map indexProviders, PageCache pageCache, - RecordFormats format ) + RecordFormats format, LogTailScanner tailScanner ) { this.progressMonitor = progressMonitor; this.fs = fs; @@ -68,6 +70,7 @@ public DatabaseMigrator( this.indexProviders = indexProviders; this.pageCache = pageCache; this.format = format; + this.tailScanner = tailScanner; } /** @@ -79,8 +82,7 @@ public void migrate( File storeDir ) { LogProvider logProvider = logService.getInternalLogProvider(); UpgradableDatabase upgradableDatabase = - new UpgradableDatabase( fs, new StoreVersionCheck( pageCache ), - format ); + new UpgradableDatabase( new StoreVersionCheck( pageCache ), format, tailScanner ); StoreUpgrader storeUpgrader = new StoreUpgrader( upgradableDatabase, progressMonitor, config, fs, pageCache, logProvider ); diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/storemigration/UpgradableDatabase.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/storemigration/UpgradableDatabase.java index 84ee4aea4082b..7558b74dec0bc 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/storemigration/UpgradableDatabase.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/storemigration/UpgradableDatabase.java @@ -20,9 +20,7 @@ package org.neo4j.kernel.impl.storemigration; import java.io.File; -import java.io.IOException; -import org.neo4j.io.fs.FileSystemAbstraction; import org.neo4j.kernel.impl.store.MetaDataStore; import org.neo4j.kernel.impl.store.format.FormatFamily; import org.neo4j.kernel.impl.store.format.RecordFormatSelector; @@ -35,10 +33,6 @@ import org.neo4j.kernel.impl.storemigration.StoreVersionCheck.Result; import org.neo4j.kernel.impl.storemigration.StoreVersionCheck.Result.Outcome; import org.neo4j.kernel.impl.transaction.log.LogTailScanner; -import org.neo4j.kernel.impl.transaction.log.PhysicalLogFiles; -import org.neo4j.kernel.impl.transaction.log.ReadableClosablePositionAwareChannel; -import org.neo4j.kernel.impl.transaction.log.entry.LogEntryReader; -import org.neo4j.kernel.impl.transaction.log.entry.VersionAwareLogEntryReader; /** * Logic to check whether a database version is upgradable to the current version. It looks at the @@ -46,16 +40,16 @@ */ public class UpgradableDatabase { - private final FileSystemAbstraction fs; private final StoreVersionCheck storeVersionCheck; private final RecordFormats format; + private final LogTailScanner tailScanner; - public UpgradableDatabase( FileSystemAbstraction fs, - StoreVersionCheck storeVersionCheck, RecordFormats format ) + public UpgradableDatabase( StoreVersionCheck storeVersionCheck, RecordFormats format, + LogTailScanner tailScanner ) { - this.fs = fs; this.storeVersionCheck = storeVersionCheck; this.format = format; + this.tailScanner = tailScanner; } /** @@ -103,7 +97,7 @@ public RecordFormats checkUpgradeable( File storeDirectory ) } else { - result = checkCleanShutDownByCheckPoint( storeDirectory ); + result = checkCleanShutDownByCheckPoint(); if ( result.outcome.isSuccessful() ) { return fromFormat; @@ -133,22 +127,17 @@ public RecordFormats checkUpgradeable( File storeDirectory ) } } - private Result checkCleanShutDownByCheckPoint( File storeDirectory ) + private Result checkCleanShutDownByCheckPoint() { // check version - PhysicalLogFiles logFiles = new PhysicalLogFiles( storeDirectory, fs ); - LogEntryReader logEntryReader = new VersionAwareLogEntryReader<>(); - LogTailScanner logTailScanner = - new LogTailScanner( logFiles, fs, logEntryReader ); try { - LogTailScanner.LogTailInformation logTailInformation = logTailScanner.find( logFiles.getHighestLogVersion() ); - if ( !logTailInformation.commitsAfterLastCheckPoint ) + if ( !tailScanner.getTailInformation().commitsAfterLastCheckPoint ) { return new Result( Result.Outcome.ok, null, null ); } } - catch ( IOException e ) + catch ( Throwable throwable ) { // ignore exception and return db not cleanly shutdown } diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/transaction/log/LogTailScanner.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/transaction/log/LogTailScanner.java index 12bafc1b6adb8..a826925df6042 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/transaction/log/LogTailScanner.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/transaction/log/LogTailScanner.java @@ -22,6 +22,7 @@ import java.io.IOException; import org.neo4j.io.fs.FileSystemAbstraction; +import org.neo4j.kernel.impl.store.UnderlyingStorageException; import org.neo4j.kernel.impl.transaction.log.entry.CheckPoint; import org.neo4j.kernel.impl.transaction.log.entry.LogEntry; import org.neo4j.kernel.impl.transaction.log.entry.LogEntryCommit; @@ -45,6 +46,7 @@ public class LogTailScanner private final PhysicalLogFiles logFiles; private final FileSystemAbstraction fileSystem; private final LogEntryReader logEntryReader; + private LogTailInformation logTailInformation; public LogTailScanner( PhysicalLogFiles logFiles, FileSystemAbstraction fileSystem, LogEntryReader logEntryReader ) @@ -54,8 +56,9 @@ public LogTailScanner( PhysicalLogFiles logFiles, FileSystemAbstraction fileSyst this.logEntryReader = logEntryReader; } - public LogTailInformation find( long fromVersionBackwards ) throws IOException + private LogTailInformation update() throws IOException { + final long fromVersionBackwards = logFiles.getHighestLogVersion(); long version = fromVersionBackwards; long versionToSearchForCommits = fromVersionBackwards; LogEntryStart latestStartEntry = null; @@ -110,8 +113,8 @@ public LogTailInformation find( long fromVersionBackwards ) throws IOException } } - // Collect data about latest entry version - if ( latestLogEntryVersion == null || version == versionToSearchForCommits ) + // Collect data about latest entry version, only in first log file + if ( version == versionToSearchForCommits || latestLogEntryVersion == null ) { latestLogEntryVersion = entry.getVersion(); } @@ -139,7 +142,7 @@ public LogTailInformation find( long fromVersionBackwards ) throws IOException : LogTailInformation.NO_TRANSACTION_ID; return new LogTailInformation( null, commitsAfterCheckPoint, firstTxAfterPosition, oldestVersionFound, - latestLogEntryVersion ); + fromVersionBackwards, latestLogEntryVersion ); } protected LogTailInformation latestCheckPoint( long fromVersionBackwards, long version, @@ -167,7 +170,7 @@ protected LogTailInformation latestCheckPoint( long fromVersionBackwards, long v ? extractFirstTxIdAfterPosition( target, fromVersionBackwards ) : LogTailInformation.NO_TRANSACTION_ID; return new LogTailInformation( latestCheckPoint, startEntryAfterCheckPoint, - firstTxIdAfterCheckPoint, oldestVersionFound, latestLogEntryVersion ); + firstTxIdAfterCheckPoint, oldestVersionFound, fromVersionBackwards, latestLogEntryVersion ); } /** @@ -217,6 +220,34 @@ protected long extractFirstTxIdAfterPosition( LogPosition initialPosition, long return LogTailInformation.NO_TRANSACTION_ID; } + /** + * Collects information about the tail of the transaction log, i.e. last checkpoint, last entry etc. + * Since this is an expensive task we do it once and reuse the result. This method is thus lazy and the first one + * calling it will take the hit. + *

+ * This is only intended to be used during startup. If you need to track the state of the tail, that can be done more + * efficiently at runtime, and this method should then only be used to restore said state. + * + * @return snapshot of the state of the transaction logs tail at startup. + * @throws UnderlyingStorageException if any errors occurs while parsing the transaction logs + */ + public LogTailInformation getTailInformation() throws UnderlyingStorageException + { + if ( logTailInformation == null ) + { + try + { + logTailInformation = update(); + } + catch ( IOException e ) + { + throw new UnderlyingStorageException( "Error encountered while parsing transaction logs", e ); + } + } + + return logTailInformation; + } + public static class LogTailInformation { public static long NO_TRANSACTION_ID = -1; @@ -225,63 +256,18 @@ public static class LogTailInformation public final boolean commitsAfterLastCheckPoint; public final long firstTxIdAfterLastCheckPoint; public final long oldestLogVersionFound; + public final long currentLogVersion; public final LogEntryVersion latestLogEntryVersion; public LogTailInformation( CheckPoint lastCheckPoint, boolean commitsAfterLastCheckPoint, - long firstTxIdAfterLastCheckPoint, long oldestLogVersionFound, LogEntryVersion latestLogEntryVersion ) + long firstTxIdAfterLastCheckPoint, long oldestLogVersionFound, long currentLogVersion, LogEntryVersion latestLogEntryVersion ) { this.lastCheckPoint = lastCheckPoint; this.commitsAfterLastCheckPoint = commitsAfterLastCheckPoint; this.firstTxIdAfterLastCheckPoint = firstTxIdAfterLastCheckPoint; this.oldestLogVersionFound = oldestLogVersionFound; + this.currentLogVersion = currentLogVersion; this.latestLogEntryVersion = latestLogEntryVersion; } - - @Override - public boolean equals( Object o ) - { - if ( this == o ) - { - return true; - } - if ( o == null || getClass() != o.getClass() ) - { - return false; - } - - LogTailInformation that = (LogTailInformation) o; - - return commitsAfterLastCheckPoint == that.commitsAfterLastCheckPoint && - firstTxIdAfterLastCheckPoint == that.firstTxIdAfterLastCheckPoint && - oldestLogVersionFound == that.oldestLogVersionFound && - (lastCheckPoint == null ? that.lastCheckPoint == null : lastCheckPoint - .equals( that.lastCheckPoint )) && - latestLogEntryVersion.equals( that.latestLogEntryVersion ); - } - - @Override - public int hashCode() - { - int result = lastCheckPoint != null ? lastCheckPoint.hashCode() : 0; - result = 31 * result + (commitsAfterLastCheckPoint ? 1 : 0); - if ( commitsAfterLastCheckPoint ) - { - result = 31 * result + Long.hashCode( firstTxIdAfterLastCheckPoint ); - } - result = 31 * result + Long.hashCode( oldestLogVersionFound ); - result = 31 * result + latestLogEntryVersion.hashCode(); - return result; - } - - @Override - public String toString() - { - return "LogTailInformation{" + - "lastCheckPoint=" + lastCheckPoint + - ", commitsAfterLastCheckPoint=" + commitsAfterLastCheckPoint + - (commitsAfterLastCheckPoint ? ", firstTxIdAfterLastCheckPoint=" + firstTxIdAfterLastCheckPoint : "") + - ", oldestLogVersionFound=" + oldestLogVersionFound + - '}'; - } } } diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/transaction/log/LogVersionUpgradeChecker.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/transaction/log/LogVersionUpgradeChecker.java new file mode 100644 index 0000000000000..f66975fd9e84e --- /dev/null +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/transaction/log/LogVersionUpgradeChecker.java @@ -0,0 +1,54 @@ +/* + * Copyright (c) 2002-2017 "Neo Technology," + * Network Engine for Objects in Lund AB [http://neotechnology.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.kernel.impl.transaction.log; + +import org.neo4j.graphdb.factory.GraphDatabaseSettings; +import org.neo4j.kernel.configuration.Config; +import org.neo4j.kernel.impl.storemigration.UpgradeNotAllowedByConfigurationException; +import org.neo4j.kernel.impl.transaction.log.entry.LogEntryVersion; + +/** + * Here we check the latest entry in the transaction log and make sure it matches the current version, if this check + * fails, it means that we will write entries with a version not compatible with the previous version responsible for + * creating the transaction logs. + *

+ * This can be considered an upgrade since the user is not able to revert back to the previous version of neo4j. This + * will effectively guard the users from accidental upgrades. + */ +public class LogVersionUpgradeChecker +{ + private LogVersionUpgradeChecker() + { + throw new AssertionError( "No instances allowed" ); + } + + public static void check( LogTailScanner tailScanner, Config config ) throws UpgradeNotAllowedByConfigurationException + { + if ( !config.get( GraphDatabaseSettings.allow_store_upgrade ) ) + { + // The user doesn't want us to upgrade the store. + LogEntryVersion latestLogEntryVersion = tailScanner.getTailInformation().latestLogEntryVersion; + if ( latestLogEntryVersion != null && LogEntryVersion.moreRecentVersionExists( latestLogEntryVersion ) ) + { + throw new UpgradeNotAllowedByConfigurationException(); + } + } + } +} diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/transaction/log/entry/LogEntryVersion.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/transaction/log/entry/LogEntryVersion.java index b9b8f7b8d8e7d..3c5805f05ec9b 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/transaction/log/entry/LogEntryVersion.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/transaction/log/entry/LogEntryVersion.java @@ -71,23 +71,24 @@ public enum LogEntryVersion V2_3( -5, LogEntryParsersV2_3.class ), V3_0( -6, LogEntryParsersV2_3.class ), // as of 2016-05-30: neo4j 2.3.5 legacy index IndexDefineCommand maps write size as short instead of byte - // See comment for V2.2.10 for version number explanation // log entry layout hasn't changed since 2_3 so just use that one V2_3_5( -8, LogEntryParsersV2_3.class ), // as of 2016-05-30: neo4j 3.0.2 legacy index IndexDefineCommand maps write size as short instead of byte - // See comment for V2.2.10 for version number explanation // log entry layout hasn't changed since 2_3 so just use that one V3_0_2( -9, LogEntryParsersV2_3.class ), // as of 2017-05-26: the records in command log entries include a bit that specifies if the command is serialised // using a fixed-width reference format, or not. This change is technically backwards compatible, so we bump the // log version to prevent mixed-version clusters from forming. V3_0_10( -10, LogEntryParsersV2_3.class ); + // Method moreRecentVersionExists() relies on the fact that we have negative numbers, thus next version to use is -11 public static final LogEntryVersion CURRENT = V3_0_10; + private static final byte LOWEST_VERSION = (byte)-V2_3.byteCode(); private static final LogEntryVersion[] ALL = values(); - private static final LogEntryVersion[] LOOKUP_BY_VERSION = new LogEntryVersion[11]; // pessimistic size + private static final LogEntryVersion[] LOOKUP_BY_VERSION; static { + LOOKUP_BY_VERSION = new LogEntryVersion[(-CURRENT.byteCode()) + 1]; // pessimistic size for ( LogEntryVersion version : ALL ) { put( LOOKUP_BY_VERSION, -version.byteCode(), version ); @@ -131,6 +132,17 @@ public LogEntryParser entryParser( byte type ) return candidate; } + /** + * Check if a more recent version of the log entry format exists and can be handled. + * + * @param version to compare against latest version + * @return {@code true} if a more recent log entry version exists + */ + public static boolean moreRecentVersionExists( LogEntryVersion version ) + { + return version.version > CURRENT.version; // reverted do to negative version numbers + } + /** * Return the correct {@link LogEntryVersion} for the given {@code version} code read from e.g. a log entry. * Lookup is fast and can be made inside critical paths, no need for externally caching the returned @@ -140,13 +152,25 @@ public LogEntryParser entryParser( byte type ) */ public static LogEntryVersion byVersion( byte version ) { - byte flattenedVersion = (byte) -version; + byte positiveVersion = (byte) -version; - if ( flattenedVersion >= 0 && flattenedVersion < LOOKUP_BY_VERSION.length ) + if ( positiveVersion >= LOWEST_VERSION && positiveVersion < LOOKUP_BY_VERSION.length ) + { + return LOOKUP_BY_VERSION[positiveVersion]; + } + byte positiveCurrentVersion = (byte) -CURRENT.byteCode(); + if ( positiveVersion > positiveCurrentVersion ) { - return LOOKUP_BY_VERSION[flattenedVersion]; + throw new IllegalArgumentException( String.format( + "Transaction logs contains entries with prefix %d, and the highest supported prefix is %d. This " + + "indicates that the log files originates from a newer version of neo4j.", + positiveVersion, positiveCurrentVersion ) ); } - throw new IllegalArgumentException( "Unrecognized log entry version " + version ); + throw new IllegalArgumentException( String.format( + "Transaction logs contains entries with prefix %d, and the lowest supported prefix is %d. This " + + "indicates that the log files originates from an older version of neo4j, which we don't support " + + "migrations from.", + positiveVersion, LOWEST_VERSION ) ); } private static void put( LogEntryVersion[] array, int index, LogEntryVersion version ) diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/transaction/log/entry/LogEntryWriter.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/transaction/log/entry/LogEntryWriter.java index da9822deff389..590e093817781 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/transaction/log/entry/LogEntryWriter.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/transaction/log/entry/LogEntryWriter.java @@ -37,17 +37,34 @@ public class LogEntryWriter { private final FlushableChannel channel; + private final LogEntryVersion version; private final Visitor serializer; + /** + * Create a writer that uses {@link LogEntryVersion#CURRENT} for versioning. + * @param channel underlying channel + */ public LogEntryWriter( FlushableChannel channel ) + { + this(channel, CURRENT ); + } + + /** + * Create a writer that uses a different version than {@link LogEntryVersion#CURRENT}. Useful when writing test for + * migration/upgrade scenarios. + * @param channel underlying channel + * @param version version to put in the header + */ + public LogEntryWriter( FlushableChannel channel, LogEntryVersion version ) { this.channel = channel; + this.version = version; this.serializer = new StorageCommandSerializer( channel ); } private void writeLogEntryHeader( byte type ) throws IOException { - channel.put( CURRENT.byteCode() ).put( type ); + channel.put( version.byteCode() ).put( type ); } public void writeStartEntry( int masterId, int authorId, long timeWritten, long latestCommittedTxWhenStarted, diff --git a/community/kernel/src/main/java/org/neo4j/kernel/recovery/DefaultRecoverySPI.java b/community/kernel/src/main/java/org/neo4j/kernel/recovery/DefaultRecoverySPI.java index 1b1bcb8ebf12f..9d721ab248b23 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/recovery/DefaultRecoverySPI.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/recovery/DefaultRecoverySPI.java @@ -28,7 +28,6 @@ import org.neo4j.kernel.impl.transaction.TransactionRepresentation; import org.neo4j.kernel.impl.transaction.log.LogPosition; import org.neo4j.kernel.impl.transaction.log.LogTailScanner; -import org.neo4j.kernel.impl.transaction.log.LogVersionRepository; import org.neo4j.kernel.impl.transaction.log.LogicalTransactionStore; import org.neo4j.kernel.impl.transaction.log.PhysicalLogFiles; import org.neo4j.kernel.impl.transaction.log.TransactionCursor; @@ -42,7 +41,6 @@ public class DefaultRecoverySPI implements Recovery.SPI { - private final LogVersionRepository logVersionRepository; private final PositionToRecoverFrom positionToRecoverFrom; private final PhysicalLogFiles logFiles; private final FileSystemAbstraction fs; @@ -53,14 +51,13 @@ public class DefaultRecoverySPI implements Recovery.SPI public DefaultRecoverySPI( StorageEngine storageEngine, PhysicalLogFiles logFiles, FileSystemAbstraction fs, - LogVersionRepository logVersionRepository, LogTailScanner logTailScanner, + LogTailScanner logTailScanner, TransactionIdStore transactionIdStore, LogicalTransactionStore logicalTransactionStore, PositionToRecoverFrom.Monitor monitor ) { this.storageEngine = storageEngine; this.logFiles = logFiles; this.fs = fs; - this.logVersionRepository = logVersionRepository; this.transactionIdStore = transactionIdStore; this.logicalTransactionStore = logicalTransactionStore; this.positionToRecoverFrom = new PositionToRecoverFrom( logTailScanner, monitor ); @@ -69,7 +66,7 @@ public DefaultRecoverySPI( @Override public LogPosition getPositionToRecoverFrom() throws IOException { - return positionToRecoverFrom.apply( logVersionRepository.getCurrentLogVersion() ); + return positionToRecoverFrom.get(); } @Override diff --git a/community/kernel/src/main/java/org/neo4j/kernel/recovery/PositionToRecoverFrom.java b/community/kernel/src/main/java/org/neo4j/kernel/recovery/PositionToRecoverFrom.java index d678a1ed2d01b..55f323ffe95a3 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/recovery/PositionToRecoverFrom.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/recovery/PositionToRecoverFrom.java @@ -21,7 +21,7 @@ import java.io.IOException; -import org.neo4j.function.ThrowingLongFunction; +import org.neo4j.function.ThrowingSupplier; import org.neo4j.kernel.impl.store.UnderlyingStorageException; import org.neo4j.kernel.impl.transaction.log.LogPosition; import org.neo4j.kernel.impl.transaction.log.LogTailScanner; @@ -31,7 +31,7 @@ /** * Utility class to find the log position to start recovery from */ -public class PositionToRecoverFrom implements ThrowingLongFunction +public class PositionToRecoverFrom implements ThrowingSupplier { public interface Monitor { @@ -79,15 +79,14 @@ public PositionToRecoverFrom( LogTailScanner logTailScanner, Monitor monitor ) /** * Find the log position to start recovery from * - * @param currentLogVersion the latest transaction log version * @return {@link LogPosition#UNSPECIFIED} if there is no need to recover otherwise the {@link LogPosition} to * start recovery from * @throws IOException if log files cannot be read */ @Override - public LogPosition apply( long currentLogVersion ) throws IOException + public LogPosition get() throws IOException { - LogTailScanner.LogTailInformation logTailInformation = logTailScanner.find( currentLogVersion ); + LogTailScanner.LogTailInformation logTailInformation = logTailScanner.getTailInformation(); if ( !logTailInformation.commitsAfterLastCheckPoint ) { monitor.noCommitsAfterLastCheckPoint( @@ -107,7 +106,7 @@ public LogPosition apply( long currentLogVersion ) throws IOException { long fromLogVersion = Math.max( INITIAL_LOG_VERSION, logTailInformation.oldestLogVersionFound ); throw new UnderlyingStorageException( "No check point found in any log file from version " + - fromLogVersion + " to " + currentLogVersion ); + fromLogVersion + " to " + logTailInformation.currentLogVersion ); } monitor.noCheckPointFound(); return LogPosition.start( 0 ); diff --git a/community/kernel/src/main/java/org/neo4j/storageengine/api/StorageEngine.java b/community/kernel/src/main/java/org/neo4j/storageengine/api/StorageEngine.java index 09846d0d03a25..237abe65b08ab 100644 --- a/community/kernel/src/main/java/org/neo4j/storageengine/api/StorageEngine.java +++ b/community/kernel/src/main/java/org/neo4j/storageengine/api/StorageEngine.java @@ -84,12 +84,6 @@ void createCommands( */ void apply( CommandsToApply batch, TransactionApplicationMode mode ) throws Exception; - /** - * @return a {@link CommandReaderFactory} capable of returning {@link CommandReader commands readers} - * for specific log entry versions. - */ - CommandReaderFactory commandReaderFactory(); - /** * Flushes and forces all changes down to underlying storage. This is a blocking call and when it returns * all changes applied to this storage engine will be durable. diff --git a/community/kernel/src/test/java/org/neo4j/kernel/RecoveryTest.java b/community/kernel/src/test/java/org/neo4j/kernel/RecoveryTest.java index 139b5db4c6a41..f24f3cc6408ad 100644 --- a/community/kernel/src/test/java/org/neo4j/kernel/RecoveryTest.java +++ b/community/kernel/src/test/java/org/neo4j/kernel/RecoveryTest.java @@ -138,17 +138,17 @@ public void shouldRecoverExistingData() throws Exception { StorageEngine storageEngine = mock( StorageEngine.class ); final LogEntryReader reader = new VersionAwareLogEntryReader<>(); - LogTailScanner finder = new LogTailScanner( logFiles, fileSystemRule.get(), reader ); + LogTailScanner tailScanner = new LogTailScanner( logFiles, fileSystemRule.get(), reader ); LogHeaderCache logHeaderCache = new LogHeaderCache( 10 ); TransactionMetadataCache metadataCache = new TransactionMetadataCache( 100 ); LogFile logFile = life.add( new PhysicalLogFile( fileSystemRule.get(), logFiles, 50, - () -> transactionIdStore.getLastCommittedTransactionId(), logVersionRepository, + transactionIdStore::getLastCommittedTransactionId, logVersionRepository, mock( PhysicalLogFile.Monitor.class ), logHeaderCache ) ); LogicalTransactionStore txStore = new PhysicalLogicalTransactionStore( logFile, metadataCache, reader ); life.add( new Recovery( new DefaultRecoverySPI( storageEngine, logFiles, fileSystemRule.get(), - logVersionRepository, finder, transactionIdStore, txStore, NO_MONITOR ) + tailScanner, transactionIdStore, txStore, NO_MONITOR ) { private int nr; @@ -241,17 +241,17 @@ public void shouldSeeThatACleanDatabaseShouldNotRequireRecovery() throws Excepti { StorageEngine storageEngine = mock( StorageEngine.class ); final LogEntryReader reader = new VersionAwareLogEntryReader<>(); - LogTailScanner finder = new LogTailScanner( logFiles, fileSystemRule.get(), reader ); + LogTailScanner tailScanner = new LogTailScanner( logFiles, fileSystemRule.get(), reader ); TransactionMetadataCache metadataCache = new TransactionMetadataCache( 100 ); LogHeaderCache logHeaderCache = new LogHeaderCache( 10 ); LogFile logFile = life.add( new PhysicalLogFile( fileSystemRule.get(), logFiles, 50, - () -> transactionIdStore.getLastCommittedTransactionId(), logVersionRepository, + transactionIdStore::getLastCommittedTransactionId, logVersionRepository, mock( PhysicalLogFile.Monitor.class ), logHeaderCache ) ); LogicalTransactionStore txStore = new PhysicalLogicalTransactionStore( logFile, metadataCache, reader ); life.add( new Recovery( new DefaultRecoverySPI( storageEngine, logFiles, fileSystemRule.get(), - logVersionRepository, finder, transactionIdStore, txStore, NO_MONITOR ) + tailScanner, transactionIdStore, txStore, NO_MONITOR ) { @Override public void startRecovery() @@ -379,17 +379,17 @@ private boolean recover( PhysicalLogFiles logFiles ) { StorageEngine storageEngine = mock( StorageEngine.class ); final LogEntryReader reader = new VersionAwareLogEntryReader<>(); - LogTailScanner finder = new LogTailScanner( logFiles, fileSystemRule.get(), reader ); + LogTailScanner tailScanner = new LogTailScanner( logFiles, fileSystemRule.get(), reader ); TransactionMetadataCache metadataCache = new TransactionMetadataCache( 100 ); LogHeaderCache logHeaderCache = new LogHeaderCache( 10 ); LogFile logFile = life.add( new PhysicalLogFile( fileSystemRule.get(), logFiles, 50, - () -> transactionIdStore.getLastCommittedTransactionId(), logVersionRepository, + transactionIdStore::getLastCommittedTransactionId, logVersionRepository, mock( PhysicalLogFile.Monitor.class ), logHeaderCache ) ); LogicalTransactionStore txStore = new PhysicalLogicalTransactionStore( logFile, metadataCache, reader ); life.add( new Recovery( new DefaultRecoverySPI( storageEngine, logFiles, fileSystemRule.get(), - logVersionRepository, finder, transactionIdStore, txStore, NO_MONITOR ) + tailScanner, transactionIdStore, txStore, NO_MONITOR ) { @Override public void startRecovery() diff --git a/community/kernel/src/test/java/org/neo4j/kernel/impl/store/TestStoreAccess.java b/community/kernel/src/test/java/org/neo4j/kernel/impl/store/TestStoreAccess.java index ddc00b0f30f65..c5b887f484a27 100644 --- a/community/kernel/src/test/java/org/neo4j/kernel/impl/store/TestStoreAccess.java +++ b/community/kernel/src/test/java/org/neo4j/kernel/impl/store/TestStoreAccess.java @@ -47,7 +47,7 @@ public class TestStoreAccess private final File storeDir = new File( "dir" ).getAbsoluteFile(); @Test - public void openingThroughStoreAccessShouldNotTriggerRecovery() throws Exception + public void openingThroughStoreAccessShouldNotTriggerRecovery() throws Throwable { try ( EphemeralFileSystemAbstraction snapshot = produceUncleanStore() ) { diff --git a/community/kernel/src/test/java/org/neo4j/kernel/impl/storemigration/MigrationTestUtils.java b/community/kernel/src/test/java/org/neo4j/kernel/impl/storemigration/MigrationTestUtils.java index 38c9b65507a39..679c82c9d760e 100644 --- a/community/kernel/src/test/java/org/neo4j/kernel/impl/storemigration/MigrationTestUtils.java +++ b/community/kernel/src/test/java/org/neo4j/kernel/impl/storemigration/MigrationTestUtils.java @@ -184,8 +184,8 @@ public static void removeCheckPointFromTxLog( FileSystemAbstraction fileSystem, { PhysicalLogFiles logFiles = new PhysicalLogFiles( workingDirectory, fileSystem ); LogEntryReader logEntryReader = new VersionAwareLogEntryReader<>(); - LogTailScanner finder = new LogTailScanner( logFiles, fileSystem, logEntryReader ); - LogTailScanner.LogTailInformation logTailInformation = finder.find( logFiles.getHighestLogVersion() ); + LogTailScanner tailScanner = new LogTailScanner( logFiles, fileSystem, logEntryReader ); + LogTailScanner.LogTailInformation logTailInformation = tailScanner.getTailInformation(); if ( logTailInformation.commitsAfterLastCheckPoint ) { diff --git a/community/kernel/src/test/java/org/neo4j/kernel/impl/storemigration/participant/StoreMigratorIT.java b/community/kernel/src/test/java/org/neo4j/kernel/impl/storemigration/participant/StoreMigratorIT.java index fdda43270041b..1c25e493463ed 100644 --- a/community/kernel/src/test/java/org/neo4j/kernel/impl/storemigration/participant/StoreMigratorIT.java +++ b/community/kernel/src/test/java/org/neo4j/kernel/impl/storemigration/participant/StoreMigratorIT.java @@ -33,9 +33,7 @@ import org.neo4j.graphdb.factory.GraphDatabaseSettings; import org.neo4j.io.fs.FileSystemAbstraction; import org.neo4j.io.pagecache.PageCache; -import org.neo4j.kernel.api.index.SchemaIndexProvider; import org.neo4j.kernel.configuration.Config; -import org.neo4j.kernel.impl.api.index.inmemory.InMemoryIndexProvider; import org.neo4j.kernel.impl.logging.LogService; import org.neo4j.kernel.impl.logging.NullLogService; import org.neo4j.kernel.impl.logging.SimpleLogService; @@ -50,6 +48,10 @@ import org.neo4j.kernel.impl.storemigration.UpgradableDatabase; import org.neo4j.kernel.impl.storemigration.monitoring.SilentMigrationProgressMonitor; import org.neo4j.kernel.impl.transaction.log.LogPosition; +import org.neo4j.kernel.impl.transaction.log.LogTailScanner; +import org.neo4j.kernel.impl.transaction.log.PhysicalLogFile; +import org.neo4j.kernel.impl.transaction.log.PhysicalLogFiles; +import org.neo4j.kernel.impl.transaction.log.entry.VersionAwareLogEntryReader; import org.neo4j.logging.AssertableLogProvider; import org.neo4j.test.rule.PageCacheRule; import org.neo4j.test.rule.TestDirectory; @@ -70,7 +72,6 @@ public class StoreMigratorIT @Rule public RuleChain ruleChain = RuleChain.outerRule( directory ).around( fileSystemRule ).around( pageCacheRule ); - private final SchemaIndexProvider schemaIndexProvider = new InMemoryIndexProvider(); private final FileSystemAbstraction fs = fileSystemRule.get(); @Parameterized.Parameter( 0 ) @@ -109,8 +110,9 @@ public void shouldBeAbleToResumeMigrationOnMoving() throws Exception // and a state of the migration saying that it has done the actual migration LogService logService = NullLogService.getInstance(); PageCache pageCache = pageCacheRule.getPageCache( fs ); - UpgradableDatabase upgradableDatabase = new UpgradableDatabase( fs, new StoreVersionCheck( pageCache ), - selectFormat() ); + LogTailScanner tailScanner = getTailScanner( storeDirectory ); + UpgradableDatabase upgradableDatabase = new UpgradableDatabase( new StoreVersionCheck( pageCache ), + selectFormat(), tailScanner ); String versionToMigrateFrom = upgradableDatabase.checkUpgradeable( storeDirectory ).storeVersion(); SilentMigrationProgressMonitor progressMonitor = new SilentMigrationProgressMonitor(); @@ -149,8 +151,9 @@ public void shouldBeAbleToMigrateWithoutErrors() throws Exception AssertableLogProvider logProvider = new AssertableLogProvider( true ); LogService logService = new SimpleLogService( logProvider, logProvider ); PageCache pageCache = pageCacheRule.getPageCache( fs ); - UpgradableDatabase upgradableDatabase = new UpgradableDatabase( fs, new StoreVersionCheck( pageCache ), - selectFormat() ); + LogTailScanner tailScanner = getTailScanner( storeDirectory ); + UpgradableDatabase upgradableDatabase = new UpgradableDatabase( new StoreVersionCheck( pageCache ), + selectFormat(), tailScanner ); String versionToMigrateFrom = upgradableDatabase.checkUpgradeable( storeDirectory ).storeVersion(); SilentMigrationProgressMonitor progressMonitor = new SilentMigrationProgressMonitor(); @@ -187,8 +190,9 @@ public void shouldBeAbleToResumeMigrationOnRebuildingCounts() throws Exception // and a state of the migration saying that it has done the actual migration LogService logService = NullLogService.getInstance(); PageCache pageCache = pageCacheRule.getPageCache( fs ); + LogTailScanner tailScanner = getTailScanner( storeDirectory ); UpgradableDatabase upgradableDatabase = - new UpgradableDatabase( fs, new StoreVersionCheck( pageCache ), selectFormat() ); + new UpgradableDatabase( new StoreVersionCheck( pageCache ), selectFormat(), tailScanner ); String versionToMigrateFrom = upgradableDatabase.checkUpgradeable( storeDirectory ).storeVersion(); SilentMigrationProgressMonitor progressMonitor = new SilentMigrationProgressMonitor(); @@ -224,8 +228,9 @@ public void shouldComputeTheLastTxLogPositionCorrectly() throws Throwable // and a state of the migration saying that it has done the actual migration LogService logService = NullLogService.getInstance(); PageCache pageCache = pageCacheRule.getPageCache( fs ); + LogTailScanner tailScanner = getTailScanner( storeDirectory ); UpgradableDatabase upgradableDatabase = - new UpgradableDatabase( fs, new StoreVersionCheck( pageCache ), selectFormat() ); + new UpgradableDatabase( new StoreVersionCheck( pageCache ), selectFormat(), tailScanner ); String versionToMigrateFrom = upgradableDatabase.checkUpgradeable( storeDirectory ).storeVersion(); SilentMigrationProgressMonitor progressMonitor = new SilentMigrationProgressMonitor(); @@ -251,8 +256,9 @@ public void shouldComputeTheLastTxInfoCorrectly() throws Exception // and a state of the migration saying that it has done the actual migration LogService logService = NullLogService.getInstance(); PageCache pageCache = pageCacheRule.getPageCache( fs ); + LogTailScanner tailScanner = getTailScanner( storeDirectory ); UpgradableDatabase upgradableDatabase = - new UpgradableDatabase( fs, new StoreVersionCheck( pageCache ), selectFormat() ); + new UpgradableDatabase( new StoreVersionCheck( pageCache ), selectFormat(), tailScanner ); String versionToMigrateFrom = upgradableDatabase.checkUpgradeable( storeDirectory ).storeVersion(); SilentMigrationProgressMonitor progressMonitor = new SilentMigrationProgressMonitor(); @@ -268,6 +274,12 @@ public void shouldComputeTheLastTxInfoCorrectly() throws Exception assertTrue( txIdComparator.apply( migrator.readLastTxInformation( migrationDir ) ) ); } + private LogTailScanner getTailScanner( File storeDirectory ) + { + PhysicalLogFiles logFiles = new PhysicalLogFiles( storeDirectory, PhysicalLogFile.DEFAULT_NAME, fs ); + return new LogTailScanner( logFiles, fs, new VersionAwareLogEntryReader<>() ); + } + private RecordFormats selectFormat() { return Standard.LATEST_RECORD_FORMATS; diff --git a/community/kernel/src/test/java/org/neo4j/kernel/impl/transaction/log/LogTailScannerTest.java b/community/kernel/src/test/java/org/neo4j/kernel/impl/transaction/log/LogTailScannerTest.java index 0b8cb99a9ae98..f18d1a545a962 100644 --- a/community/kernel/src/test/java/org/neo4j/kernel/impl/transaction/log/LogTailScannerTest.java +++ b/community/kernel/src/test/java/org/neo4j/kernel/impl/transaction/log/LogTailScannerTest.java @@ -59,7 +59,7 @@ public class LogTailScannerTest public final EphemeralFileSystemRule fsRule = new EphemeralFileSystemRule(); private final File directory = new File( "/somewhere" ); private final LogEntryReader reader = new VersionAwareLogEntryReader<>(); - private LogTailScanner finder; + private LogTailScanner tailScanner; private PhysicalLogFiles logFiles; private final int startLogVersion; private final int endLogVersion; @@ -82,7 +82,7 @@ public void setUp() { fsRule.get().mkdirs( directory ); logFiles = new PhysicalLogFiles( directory, fsRule.get() ); - finder = new LogTailScanner( logFiles, fsRule.get(), reader ); + tailScanner = new LogTailScanner( logFiles, fsRule.get(), reader ); } @Test @@ -90,11 +90,9 @@ public void noLogFilesFound() throws Throwable { // given no files setupLogFiles(); - int logVersion = startLogVersion; - LogTailScanner finder = new LogTailScanner( logFiles, fsRule.get(), reader ); // when - LogTailScanner.LogTailInformation logTailInformation = finder.find( logVersion ); + LogTailInformation logTailInformation = tailScanner.getTailInformation(); // then assertLatestCheckPoint( false, false, NO_TRANSACTION_ID, -1, logTailInformation ); @@ -104,29 +102,27 @@ public void noLogFilesFound() throws Throwable public void oneLogFileNoCheckPoints() throws Throwable { // given - int logVersion = endLogVersion; setupLogFiles( logFile() ); // when - LogTailScanner.LogTailInformation logTailInformation = finder.find( logVersion ); + LogTailInformation logTailInformation = tailScanner.getTailInformation(); // then - assertLatestCheckPoint( false, false, NO_TRANSACTION_ID, logVersion, logTailInformation ); + assertLatestCheckPoint( false, false, NO_TRANSACTION_ID, endLogVersion, logTailInformation ); } @Test public void oneLogFileNoCheckPointsOneStart() throws Throwable { // given - int logVersion = endLogVersion; long txId = 10; setupLogFiles( logFile( start(), commit( txId ) ) ); // when - LogTailScanner.LogTailInformation logTailInformation = finder.find( logVersion ); + LogTailInformation logTailInformation = tailScanner.getTailInformation(); // then - assertLatestCheckPoint( false, true, txId, logVersion, logTailInformation ); + assertLatestCheckPoint( false, true, txId, endLogVersion, logTailInformation ); } @Test @@ -136,7 +132,7 @@ public void twoLogFilesNoCheckPoints() throws Throwable setupLogFiles( logFile(), logFile() ); // when - LogTailScanner.LogTailInformation logTailInformation = finder.find( endLogVersion ); + LogTailInformation logTailInformation = tailScanner.getTailInformation(); // then assertLatestCheckPoint( false, false, NO_TRANSACTION_ID, startLogVersion, logTailInformation ); @@ -150,7 +146,7 @@ public void twoLogFilesNoCheckPointsOneStart() throws Throwable setupLogFiles( logFile(), logFile( start(), commit( txId ) ) ); // when - LogTailScanner.LogTailInformation logTailInformation = finder.find( endLogVersion ); + LogTailInformation logTailInformation = tailScanner.getTailInformation(); // then assertLatestCheckPoint( false, true, txId, startLogVersion, logTailInformation ); @@ -163,7 +159,7 @@ public void twoLogFilesNoCheckPointsOneStartWithoutCommit() throws Throwable setupLogFiles( logFile(), logFile( start() ) ); // when - LogTailScanner.LogTailInformation logTailInformation = finder.find( endLogVersion ); + LogTailInformation logTailInformation = tailScanner.getTailInformation(); // then assertLatestCheckPoint( false, true, NO_TRANSACTION_ID, startLogVersion, logTailInformation ); @@ -177,7 +173,7 @@ public void twoLogFilesNoCheckPointsTwoCommits() throws Throwable setupLogFiles( logFile(), logFile( start(), commit( txId ), start(), commit( txId + 1 ) ) ); // when - LogTailScanner.LogTailInformation logTailInformation = finder.find( endLogVersion ); + LogTailInformation logTailInformation = tailScanner.getTailInformation(); // then assertLatestCheckPoint( false, true, txId, startLogVersion, logTailInformation ); @@ -195,7 +191,7 @@ public void twoLogFilesCheckPointTargetsPrevious() throws Exception logFile( checkPoint( position ) ) ); // when - LogTailInformation logTailInformation = finder.find( endLogVersion ); + LogTailInformation logTailInformation = tailScanner.getTailInformation(); // then assertLatestCheckPoint( true, true, txId, endLogVersion, logTailInformation ); @@ -208,7 +204,7 @@ public void latestLogFileContainingACheckPointOnly() throws Throwable setupLogFiles( logFile( checkPoint() ) ); // when - LogTailInformation logTailInformation = finder.find( endLogVersion ); + LogTailInformation logTailInformation = tailScanner.getTailInformation(); // then assertLatestCheckPoint( true, false, NO_TRANSACTION_ID, endLogVersion, logTailInformation ); @@ -221,7 +217,7 @@ public void latestLogFileContainingACheckPointAndAStartBefore() throws Throwable setupLogFiles( logFile( start(), checkPoint() ) ); // when - LogTailScanner.LogTailInformation logTailInformation = finder.find( endLogVersion ); + LogTailInformation logTailInformation = tailScanner.getTailInformation(); // then assertLatestCheckPoint( true, false, NO_TRANSACTION_ID, endLogVersion, logTailInformation ); @@ -232,13 +228,13 @@ public void bigFileLatestCheckpointFindsStartAfter() throws Throwable { long firstTxAfterCheckpoint = Integer.MAX_VALUE + 4L; - LogTailScanner checkPointFinder = - new FirstTxIdConfigurableCheckpointFinder( firstTxAfterCheckpoint, logFiles, fsRule.get(), reader ); + LogTailScanner tailScanner = + new FirstTxIdConfigurableTailScanner( firstTxAfterCheckpoint, logFiles, fsRule.get(), reader ); LogEntryStart startEntry = new LogEntryStart( 1, 2, 3L, 4L, new byte[]{5, 6}, new LogPosition( endLogVersion, Integer.MAX_VALUE + 17L ) ); CheckPoint checkPoint = new CheckPoint( new LogPosition( endLogVersion, 16L ) ); LogTailInformation - logTailInformation = checkPointFinder.latestCheckPoint( endLogVersion, endLogVersion, startEntry, + logTailInformation = tailScanner.latestCheckPoint( endLogVersion, endLogVersion, startEntry, endLogVersion, checkPoint, latestLogEntryVersion ); assertLatestCheckPoint( true, true, firstTxAfterCheckpoint, endLogVersion, logTailInformation ); @@ -253,7 +249,7 @@ public void latestLogFileContainingACheckPointAndAStartAfter() throws Throwable setupLogFiles( logFile( start, commit( txId ), checkPoint( start ) ) ); // when - LogTailScanner.LogTailInformation logTailInformation = finder.find( endLogVersion ); + LogTailInformation logTailInformation = tailScanner.getTailInformation(); // then assertLatestCheckPoint( true, true, txId, endLogVersion, logTailInformation ); @@ -267,7 +263,7 @@ public void latestLogFileContainingACheckPointAndAStartWithoutCommitAfter() thro setupLogFiles( logFile( start, checkPoint( start ) ) ); // when - LogTailScanner.LogTailInformation logTailInformation = finder.find( endLogVersion ); + LogTailInformation logTailInformation = tailScanner.getTailInformation(); // then assertLatestCheckPoint( true, true, NO_TRANSACTION_ID, endLogVersion, logTailInformation ); @@ -280,7 +276,7 @@ public void latestLogFileContainingMultipleCheckPointsOneStartInBetween() throws setupLogFiles( logFile( checkPoint(), start(), checkPoint() ) ); // when - LogTailScanner.LogTailInformation logTailInformation = finder.find( endLogVersion ); + LogTailInformation logTailInformation = tailScanner.getTailInformation(); // then assertLatestCheckPoint( true, false, NO_TRANSACTION_ID, endLogVersion, logTailInformation ); @@ -294,7 +290,7 @@ public void latestLogFileContainingMultipleCheckPointsOneStartAfterBoth() throws setupLogFiles( logFile( checkPoint(), checkPoint(), start(), commit( txId ) ) ); // when - LogTailScanner.LogTailInformation logTailInformation = finder.find( endLogVersion ); + LogTailInformation logTailInformation = tailScanner.getTailInformation(); // then assertLatestCheckPoint( true, true, txId, endLogVersion, logTailInformation ); @@ -309,7 +305,7 @@ public void olderLogFileContainingACheckPointAndNewerFileContainingAStart() thro setupLogFiles( logFile( checkPoint() ), logFile( start, commit( txId ) ) ); // when - LogTailScanner.LogTailInformation logTailInformation = finder.find( endLogVersion ); + LogTailInformation logTailInformation = tailScanner.getTailInformation(); // then assertLatestCheckPoint( true, true, txId, startLogVersion, logTailInformation ); @@ -323,7 +319,7 @@ public void olderLogFileContainingACheckPointAndNewerFileIsEmpty() throws Throwa setupLogFiles( logFile( start, checkPoint() ), logFile() ); // when - LogTailInformation logTailInformation = finder.find( endLogVersion ); + LogTailInformation logTailInformation = tailScanner.getTailInformation(); // then assertLatestCheckPoint( true, false, NO_TRANSACTION_ID, startLogVersion, logTailInformation ); @@ -338,7 +334,7 @@ public void olderLogFileContainingAStartAndNewerFileContainingACheckPointPointin setupLogFiles( logFile( start, commit( txId ) ), logFile( checkPoint( start ) ) ); // when - LogTailInformation logTailInformation = finder.find( endLogVersion ); + LogTailInformation logTailInformation = tailScanner.getTailInformation(); // then assertLatestCheckPoint( true, true, txId, endLogVersion, logTailInformation ); @@ -353,7 +349,7 @@ public void olderLogFileContainingAStartAndNewerFileContainingACheckPointPointin setupLogFiles( logFile( start ), logFile( checkPoint( start ) ) ); // when - LogTailInformation logTailInformation = finder.find( endLogVersion ); + LogTailInformation logTailInformation = tailScanner.getTailInformation(); // then assertLatestCheckPoint( true, false, NO_TRANSACTION_ID, endLogVersion, logTailInformation ); @@ -367,7 +363,7 @@ public void olderLogFileContainingAStartAndNewerFileContainingACheckPointPointin setupLogFiles( logFile( start(), commit( 3 ), position ), logFile( checkPoint( position ) ) ); // when - LogTailScanner.LogTailInformation logTailInformation = finder.find( endLogVersion ); + LogTailInformation logTailInformation = tailScanner.getTailInformation(); // then assertLatestCheckPoint( true, false, NO_TRANSACTION_ID, endLogVersion, logTailInformation ); @@ -381,7 +377,7 @@ public void latestLogEmptyStartEntryBeforeAndAfterCheckPointInTheLastButOneLog() setupLogFiles( logFile( start(), checkPoint(), start(), commit( txId ) ), logFile() ); // when - LogTailScanner.LogTailInformation logTailInformation = finder.find( endLogVersion ); + LogTailInformation logTailInformation = tailScanner.getTailInformation(); // then assertLatestCheckPoint( true, true, txId, startLogVersion, logTailInformation ); @@ -389,8 +385,7 @@ public void latestLogEmptyStartEntryBeforeAndAfterCheckPointInTheLastButOneLog() // === Below is code for helping the tests above === - @SafeVarargs - private final void setupLogFiles( LogCreator... logFiles ) throws IOException + private void setupLogFiles( LogCreator... logFiles ) throws IOException { Map positions = new HashMap<>(); long version = endLogVersion - logFiles.length; @@ -528,7 +523,7 @@ private static class PositionEntry implements Entry } private void assertLatestCheckPoint( boolean hasCheckPointEntry, boolean commitsAfterLastCheckPoint, - long firstTxIdAfterLastCheckPoint, long logVersion, LogTailScanner.LogTailInformation logTailInformation ) + long firstTxIdAfterLastCheckPoint, long logVersion, LogTailInformation logTailInformation ) { assertEquals( hasCheckPointEntry, logTailInformation.lastCheckPoint != null ); assertEquals( commitsAfterLastCheckPoint, logTailInformation.commitsAfterLastCheckPoint ); @@ -539,12 +534,12 @@ private void assertLatestCheckPoint( boolean hasCheckPointEntry, boolean commits assertEquals( logVersion, logTailInformation.oldestLogVersionFound ); } - private static class FirstTxIdConfigurableCheckpointFinder extends LogTailScanner + private static class FirstTxIdConfigurableTailScanner extends LogTailScanner { private final long txId; - FirstTxIdConfigurableCheckpointFinder( long txId, PhysicalLogFiles logFiles, FileSystemAbstraction fileSystem, + FirstTxIdConfigurableTailScanner( long txId, PhysicalLogFiles logFiles, FileSystemAbstraction fileSystem, LogEntryReader logEntryReader ) { super( logFiles, fileSystem, logEntryReader ); diff --git a/community/kernel/src/test/java/org/neo4j/kernel/impl/transaction/log/LogVersionUpgradeCheckerIT.java b/community/kernel/src/test/java/org/neo4j/kernel/impl/transaction/log/LogVersionUpgradeCheckerIT.java new file mode 100644 index 0000000000000..909ed19e9aec3 --- /dev/null +++ b/community/kernel/src/test/java/org/neo4j/kernel/impl/transaction/log/LogVersionUpgradeCheckerIT.java @@ -0,0 +1,152 @@ +/* + * Copyright (c) 2002-2017 "Neo Technology," + * Network Engine for Objects in Lund AB [http://neotechnology.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.kernel.impl.transaction.log; + +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; +import org.junit.rules.RuleChain; + +import java.io.File; +import java.io.IOException; +import java.util.concurrent.atomic.AtomicLong; + +import org.neo4j.graphdb.GraphDatabaseService; +import org.neo4j.graphdb.Transaction; +import org.neo4j.graphdb.factory.GraphDatabaseSettings; +import org.neo4j.io.pagecache.PageCache; +import org.neo4j.kernel.impl.storemigration.UpgradeNotAllowedByConfigurationException; +import org.neo4j.kernel.impl.transaction.log.entry.LogEntryVersion; +import org.neo4j.kernel.impl.transaction.log.entry.LogEntryWriter; +import org.neo4j.kernel.impl.transaction.log.entry.VersionAwareLogEntryReader; +import org.neo4j.kernel.lifecycle.Lifespan; +import org.neo4j.test.TestGraphDatabaseFactory; +import org.neo4j.test.matchers.NestedThrowableMatcher; +import org.neo4j.test.rule.PageCacheRule; +import org.neo4j.test.rule.TestDirectory; +import org.neo4j.test.rule.fs.DefaultFileSystemRule; + +import static org.neo4j.graphdb.Label.label; + +public class LogVersionUpgradeCheckerIT +{ + private final TestDirectory storeDirectory = TestDirectory.testDirectory(); + private final DefaultFileSystemRule fs = new DefaultFileSystemRule(); + private final PageCacheRule pageCacheRule = new PageCacheRule(); + + @Rule + public RuleChain ruleChain = RuleChain.outerRule( storeDirectory ).around( fs ).around( pageCacheRule ); + + @Rule + public ExpectedException expect = ExpectedException.none(); + + @Test + public void startAsNormalWhenUpgradeIsNotAllowed() throws Exception + { + createGraphDbAndKillIt(); + + // Try to start with upgrading disabled + final GraphDatabaseService db = new TestGraphDatabaseFactory() + .setFileSystem( fs.get() ) + .newImpermanentDatabaseBuilder( storeDirectory.graphDbDir() ) + .setConfig( GraphDatabaseSettings.allow_store_upgrade, "false" ) + .newGraphDatabase(); + db.shutdown(); + } + + @Test + public void failToStartFromOlderTransactionLogsIfNotAllowed() throws Exception + { + createStoreWithLogEntryVersion( LogEntryVersion.V2_3 ); + + expect.expect( new NestedThrowableMatcher( UpgradeNotAllowedByConfigurationException.class ) ); + + // Try to start with upgrading disabled + final GraphDatabaseService db = new TestGraphDatabaseFactory() + .setFileSystem( fs.get() ) + .newImpermanentDatabaseBuilder( storeDirectory.graphDbDir() ) + .setConfig( GraphDatabaseSettings.allow_store_upgrade, "false" ) + .newGraphDatabase(); + db.shutdown(); + } + + @Test + public void startFromOlderTransactionLogsIfAllowed() throws Exception + { + createStoreWithLogEntryVersion( LogEntryVersion.V2_3 ); + + // Try to start with upgrading enabled + final GraphDatabaseService db = new TestGraphDatabaseFactory() + .setFileSystem( fs.get() ) + .newImpermanentDatabaseBuilder( storeDirectory.graphDbDir() ) + .setConfig( GraphDatabaseSettings.allow_store_upgrade, "true" ) + .newGraphDatabase(); + db.shutdown(); + } + + private void createGraphDbAndKillIt() throws Exception + { + final GraphDatabaseService db = new TestGraphDatabaseFactory() + .setFileSystem( fs ) + .newImpermanentDatabaseBuilder( storeDirectory.graphDbDir() ) + .newGraphDatabase(); + + try ( Transaction tx = db.beginTx() ) + { + db.createNode( label( "FOO" ) ); + db.createNode( label( "BAR" ) ); + tx.success(); + } + + db.shutdown(); + } + + private void createStoreWithLogEntryVersion( LogEntryVersion logEntryVersion ) throws Exception + { + createGraphDbAndKillIt(); + appendCheckpoint( logEntryVersion ); + } + + private void appendCheckpoint( LogEntryVersion logVersion ) throws IOException + { + AtomicLong lastId = new AtomicLong(); + File storeDir = storeDirectory.graphDbDir(); + PageCache pageCache = pageCacheRule.getPageCache( fs ); + PhysicalLogFiles logFiles = new PhysicalLogFiles( storeDir, fs ); + ReadOnlyLogVersionRepository logVersionRepository = new ReadOnlyLogVersionRepository( pageCache, storeDir ); + PhysicalLogFile logFile = new PhysicalLogFile( fs, logFiles, Long.MAX_VALUE , + lastId::get, logVersionRepository, + PhysicalLogFile.NO_MONITOR, + new LogHeaderCache( 10 ) ); + + LogTailScanner tailScanner = new LogTailScanner( logFiles, fs, new VersionAwareLogEntryReader<>() ); + LogTailScanner.LogTailInformation tailInformation = tailScanner.getTailInformation(); + + try ( Lifespan lifespan = new Lifespan( logFile ) ) + { + FlushablePositionAwareChannel channel = logFile.getWriter(); + TransactionLogWriter transactionLogWriter = new TransactionLogWriter( new LogEntryWriter( channel, + logVersion ) ); + + transactionLogWriter.checkPoint( tailInformation.lastCheckPoint.getLogPosition() ); + channel.prepareForFlush().flush(); + } + } +} diff --git a/community/kernel/src/test/java/org/neo4j/kernel/impl/transaction/log/LogVersionUpgradeCheckerTest.java b/community/kernel/src/test/java/org/neo4j/kernel/impl/transaction/log/LogVersionUpgradeCheckerTest.java new file mode 100644 index 0000000000000..2a636ccea0ae5 --- /dev/null +++ b/community/kernel/src/test/java/org/neo4j/kernel/impl/transaction/log/LogVersionUpgradeCheckerTest.java @@ -0,0 +1,82 @@ +/* + * Copyright (c) 2002-2017 "Neo Technology," + * Network Engine for Objects in Lund AB [http://neotechnology.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.kernel.impl.transaction.log; + +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; + +import org.neo4j.graphdb.factory.GraphDatabaseSettings; +import org.neo4j.kernel.configuration.Config; +import org.neo4j.kernel.impl.storemigration.UpgradeNotAllowedByConfigurationException; +import org.neo4j.kernel.impl.transaction.log.entry.LogEntryVersion; + +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +public class LogVersionUpgradeCheckerTest +{ + private LogTailScanner tailScanner = mock( LogTailScanner.class ); + + @Rule + public ExpectedException expect = ExpectedException.none(); + + @Test + public void noThrowWhenLatestVersionAndUpgradeIsNotAllowed() throws Throwable + { + when( tailScanner.getTailInformation() ).thenReturn( new OnlyVersionTailInformation( LogEntryVersion.CURRENT ) ); + + LogVersionUpgradeChecker.check( tailScanner, Config.defaults( GraphDatabaseSettings.allow_store_upgrade, "false") ); + } + + @Test + public void throwWhenVersionIsOlderAndUpgradeIsNotAllowed() throws Throwable + { + when( tailScanner.getTailInformation() ).thenReturn( new OnlyVersionTailInformation( LogEntryVersion.V2_3 ) ); + + expect.expect( UpgradeNotAllowedByConfigurationException.class ); + + LogVersionUpgradeChecker.check( tailScanner, Config.defaults( GraphDatabaseSettings.allow_store_upgrade, "false") ); + } + + @Test + public void stillAcceptLatestVersionWhenUpgradeIsAllowed() throws Throwable + { + when( tailScanner.getTailInformation() ).thenReturn( new OnlyVersionTailInformation( LogEntryVersion.CURRENT ) ); + + LogVersionUpgradeChecker.check( tailScanner, Config.defaults( GraphDatabaseSettings.allow_store_upgrade, "true") ); + } + + @Test + public void acceptOlderLogsWhenUpgradeIsAllowed() throws Throwable + { + when( tailScanner.getTailInformation() ).thenReturn( new OnlyVersionTailInformation( LogEntryVersion.V2_3 ) ); + + LogVersionUpgradeChecker.check( tailScanner, Config.defaults( GraphDatabaseSettings.allow_store_upgrade, "true") ); + } + + private static class OnlyVersionTailInformation extends LogTailScanner.LogTailInformation + { + OnlyVersionTailInformation( LogEntryVersion logEntryVersion ) + { + super( null, false, 0, 0, 0, logEntryVersion ); + } + } +} diff --git a/community/kernel/src/test/java/org/neo4j/kernel/impl/transaction/log/entry/LogEntryVersionTest.java b/community/kernel/src/test/java/org/neo4j/kernel/impl/transaction/log/entry/LogEntryVersionTest.java index 82b512a511360..83b6b1c5f471a 100644 --- a/community/kernel/src/test/java/org/neo4j/kernel/impl/transaction/log/entry/LogEntryVersionTest.java +++ b/community/kernel/src/test/java/org/neo4j/kernel/impl/transaction/log/entry/LogEntryVersionTest.java @@ -19,12 +19,19 @@ */ package org.neo4j.kernel.impl.transaction.log.entry; +import org.junit.Rule; import org.junit.Test; +import org.junit.rules.ExpectedException; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; public class LogEntryVersionTest { + @Rule + public ExpectedException expect = ExpectedException.none(); + @Test public void shouldBeAbleToSelectAnyVersion() throws Exception { @@ -40,4 +47,28 @@ public void shouldBeAbleToSelectAnyVersion() throws Exception assertEquals( version, selectedVersion ); } } + + @Test + public void shouldWarnAboutOldLogVersion() throws Exception + { + expect.expect( IllegalArgumentException.class ); + LogEntryVersion.byVersion( (byte)-4 ); + } + + @Test + public void shouldWarnAboutNewerLogVersion() throws Exception + { + expect.expect( IllegalArgumentException.class ); + LogEntryVersion.byVersion( (byte)-42 ); // unused for now + } + + @Test + public void moreRecent() throws Exception + { + assertTrue( LogEntryVersion.moreRecentVersionExists( LogEntryVersion.V2_3 ) ); + assertTrue( LogEntryVersion.moreRecentVersionExists( LogEntryVersion.V3_0 ) ); + assertTrue( LogEntryVersion.moreRecentVersionExists( LogEntryVersion.V2_3_5 ) ); + assertTrue( LogEntryVersion.moreRecentVersionExists( LogEntryVersion.V3_0_2 ) ); + assertFalse( LogEntryVersion.moreRecentVersionExists( LogEntryVersion.V3_0_10 ) ); + } } diff --git a/community/kernel/src/test/java/org/neo4j/kernel/recovery/PositionToRecoverFromTest.java b/community/kernel/src/test/java/org/neo4j/kernel/recovery/PositionToRecoverFromTest.java index 81b70796fad97..687951aff275d 100644 --- a/community/kernel/src/test/java/org/neo4j/kernel/recovery/PositionToRecoverFromTest.java +++ b/community/kernel/src/test/java/org/neo4j/kernel/recovery/PositionToRecoverFromTest.java @@ -38,19 +38,20 @@ public class PositionToRecoverFromTest { + private final long currentLogVersion = 2L; private final long logVersion = 2L; - private final LogTailScanner finder = mock( LogTailScanner.class ); + private final LogTailScanner tailScanner = mock( LogTailScanner.class ); private final Monitor monitor = mock( Monitor.class ); @Test public void shouldReturnUnspecifiedIfThereIsNoNeedForRecovery() throws Throwable { // given - when( finder.find( logVersion ) ).thenReturn( new LogTailScanner.LogTailInformation( null, false, NO_TRANSACTION_ID, logVersion, - LogEntryVersion.CURRENT ) ); + when( tailScanner.getTailInformation() ).thenReturn( new LogTailScanner.LogTailInformation( null, false, NO_TRANSACTION_ID, logVersion, + currentLogVersion, LogEntryVersion.CURRENT ) ); // when - LogPosition logPosition = new PositionToRecoverFrom( finder, monitor ).apply( logVersion ); + LogPosition logPosition = new PositionToRecoverFrom( tailScanner, monitor ).get(); // then verify( monitor ).noCommitsAfterLastCheckPoint( null ); @@ -62,12 +63,12 @@ public void shouldReturnLogPositionToRecoverFromIfNeeded() throws Throwable { // given LogPosition checkPointLogPosition = new LogPosition( 1L, 4242 ); - when( finder.find( logVersion ) ) + when( tailScanner.getTailInformation() ) .thenReturn( new LogTailInformation( new CheckPoint( checkPointLogPosition ), true, 10L, logVersion, - LogEntryVersion.CURRENT ) ); + currentLogVersion, LogEntryVersion.CURRENT ) ); // when - LogPosition logPosition = new PositionToRecoverFrom( finder, monitor ).apply( logVersion ); + LogPosition logPosition = new PositionToRecoverFrom( tailScanner, monitor ).get(); // then verify( monitor ).commitsAfterLastCheckPoint( checkPointLogPosition, 10L ); @@ -78,11 +79,11 @@ public void shouldReturnLogPositionToRecoverFromIfNeeded() throws Throwable public void shouldRecoverFromStartOfLogZeroIfThereAreNoCheckPointAndOldestLogIsVersionZero() throws Throwable { // given - when( finder.find( logVersion ) ).thenReturn( new LogTailInformation( null, true, 10L, INITIAL_LOG_VERSION, - LogEntryVersion.CURRENT ) ); + when( tailScanner.getTailInformation() ).thenReturn( new LogTailInformation( null, true, 10L, INITIAL_LOG_VERSION, + currentLogVersion, LogEntryVersion.CURRENT ) ); // when - LogPosition logPosition = new PositionToRecoverFrom( finder, monitor ).apply( logVersion ); + LogPosition logPosition = new PositionToRecoverFrom( tailScanner, monitor ).get(); // then verify( monitor ).noCheckPointFound(); @@ -94,13 +95,13 @@ public void shouldFailIfThereAreNoCheckPointsAndOldestLogVersionInNotZero() thro { // given long oldestLogVersionFound = 1L; - when( finder.find( logVersion ) ).thenReturn( new LogTailScanner.LogTailInformation( null, true, 10L, oldestLogVersionFound, - LogEntryVersion.CURRENT ) ); + when( tailScanner.getTailInformation() ).thenReturn( new LogTailScanner.LogTailInformation( null, true, 10L, oldestLogVersionFound, + currentLogVersion, LogEntryVersion.CURRENT ) ); // when try { - new PositionToRecoverFrom( finder, monitor ).apply( logVersion ); + new PositionToRecoverFrom( tailScanner, monitor ).get(); } catch ( UnderlyingStorageException ex ) { diff --git a/community/neo4j/src/test/java/upgrade/StoreUpgraderInterruptionTestIT.java b/community/neo4j/src/test/java/upgrade/StoreUpgraderInterruptionTestIT.java index 80f7e4ba4aae6..e9ab123cf1f10 100644 --- a/community/neo4j/src/test/java/upgrade/StoreUpgraderInterruptionTestIT.java +++ b/community/neo4j/src/test/java/upgrade/StoreUpgraderInterruptionTestIT.java @@ -52,6 +52,9 @@ import org.neo4j.kernel.impl.storemigration.monitoring.SilentMigrationProgressMonitor; import org.neo4j.kernel.impl.storemigration.participant.SchemaIndexMigrator; import org.neo4j.kernel.impl.storemigration.participant.StoreMigrator; +import org.neo4j.kernel.impl.transaction.log.LogTailScanner; +import org.neo4j.kernel.impl.transaction.log.PhysicalLogFiles; +import org.neo4j.kernel.impl.transaction.log.entry.VersionAwareLogEntryReader; import org.neo4j.logging.NullLogProvider; import org.neo4j.test.TestGraphDatabaseFactory; import org.neo4j.test.rule.PageCacheRule; @@ -106,8 +109,7 @@ public void shouldSucceedWithUpgradeAfterPreviousAttemptDiedDuringMigration() MigrationTestUtils.prepareSampleLegacyDatabase( version, fs, workingDirectory, prepareDirectory ); PageCache pageCache = pageCacheRule.getPageCache( fs ); StoreVersionCheck check = new StoreVersionCheck( pageCache ); - UpgradableDatabase upgradableDatabase = - new UpgradableDatabase( fs, check, Standard.LATEST_RECORD_FORMATS ); + UpgradableDatabase upgradableDatabase = getUpgradableDatabase( check ); SilentMigrationProgressMonitor progressMonitor = new SilentMigrationProgressMonitor(); LogService logService = NullLogService.getInstance(); StoreMigrator failingStoreMigrator = new StoreMigrator( fs, pageCache, CONFIG, logService ) @@ -147,6 +149,13 @@ public void migrate( File sourceStoreDir, File targetStoreDir, assertConsistentStore( workingDirectory ); } + private UpgradableDatabase getUpgradableDatabase( StoreVersionCheck check ) + { + PhysicalLogFiles logFiles = new PhysicalLogFiles( workingDirectory, fs ); + LogTailScanner tailScanner = new LogTailScanner( logFiles, fs, new VersionAwareLogEntryReader<>() ); + return new UpgradableDatabase( check, Standard.LATEST_RECORD_FORMATS, tailScanner ); + } + private SchemaIndexMigrator createIndexMigrator() { return new SchemaIndexMigrator( fs, schemaIndexProvider ); @@ -161,8 +170,7 @@ public void shouldSucceedWithUpgradeAfterPreviousAttemptDiedDuringMovingFiles() MigrationTestUtils.prepareSampleLegacyDatabase( version, fs, workingDirectory, prepareDirectory ); PageCache pageCache = pageCacheRule.getPageCache( fs ); StoreVersionCheck check = new StoreVersionCheck( pageCache ); - UpgradableDatabase upgradableDatabase = - new UpgradableDatabase( fs, check, Standard.LATEST_RECORD_FORMATS ); + UpgradableDatabase upgradableDatabase = getUpgradableDatabase( check ); SilentMigrationProgressMonitor progressMonitor = new SilentMigrationProgressMonitor(); LogService logService = NullLogService.getInstance(); StoreMigrator failingStoreMigrator = new StoreMigrator( fs, pageCache, CONFIG, logService ) @@ -217,7 +225,9 @@ private StoreUpgrader newUpgrader( UpgradableDatabase upgradableDatabase, PageCa private void startStopDatabase( File workingDirectory ) { - GraphDatabaseService databaseService = new TestGraphDatabaseFactory().newEmbeddedDatabase( workingDirectory ); + GraphDatabaseService databaseService = + new TestGraphDatabaseFactory().newEmbeddedDatabaseBuilder( workingDirectory ) + .setConfig( GraphDatabaseSettings.allow_store_upgrade, "true" ).newGraphDatabase(); databaseService.shutdown(); } } diff --git a/community/neo4j/src/test/java/upgrade/StoreUpgraderTest.java b/community/neo4j/src/test/java/upgrade/StoreUpgraderTest.java index ea0eb923538e3..b222c212f63da 100644 --- a/community/neo4j/src/test/java/upgrade/StoreUpgraderTest.java +++ b/community/neo4j/src/test/java/upgrade/StoreUpgraderTest.java @@ -32,6 +32,7 @@ import java.io.IOException; import java.util.Arrays; import java.util.Collection; +import java.util.Collections; import java.util.List; import org.neo4j.graphdb.factory.GraphDatabaseSettings; @@ -60,6 +61,9 @@ import org.neo4j.kernel.impl.storemigration.participant.CountsMigrator; import org.neo4j.kernel.impl.storemigration.participant.SchemaIndexMigrator; import org.neo4j.kernel.impl.storemigration.participant.StoreMigrator; +import org.neo4j.kernel.impl.transaction.log.LogTailScanner; +import org.neo4j.kernel.impl.transaction.log.PhysicalLogFiles; +import org.neo4j.kernel.impl.transaction.log.entry.VersionAwareLogEntryReader; import org.neo4j.logging.AssertableLogProvider; import org.neo4j.logging.NullLogProvider; import org.neo4j.test.rule.PageCacheRule; @@ -114,9 +118,7 @@ public StoreUpgraderTest( String version ) @Parameterized.Parameters( name = "{0}" ) public static Collection versions() { - return Arrays.asList( - StandardV2_3.STORE_VERSION - ); + return Collections.singletonList( StandardV2_3.STORE_VERSION ); } @Before @@ -133,9 +135,7 @@ public void shouldHaltUpgradeIfUpgradeConfigurationVetoesTheProcess() throws IOE PageCache pageCache = pageCacheRule.getPageCache( fileSystem ); Config deniedMigrationConfig = Config.defaults( GraphDatabaseSettings.allow_upgrade, "false" ); - UpgradableDatabase upgradableDatabase = new UpgradableDatabase( fileSystem, - new StoreVersionCheck( pageCache ), - getRecordFormats() ); + UpgradableDatabase upgradableDatabase = getUpgradableDatabase( pageCache ); try { @@ -158,9 +158,7 @@ public void shouldRefuseToUpgradeIfAnyOfTheStoresWereNotShutDownCleanly() fileSystem.deleteRecursively( comparisonDirectory ); fileSystem.copyRecursively( dbDirectory, comparisonDirectory ); PageCache pageCache = pageCacheRule.getPageCache( fileSystem ); - UpgradableDatabase upgradableDatabase = new UpgradableDatabase( fileSystem, - new StoreVersionCheck( pageCache ), - getRecordFormats() ); + UpgradableDatabase upgradableDatabase = getUpgradableDatabase( pageCache ); try { @@ -185,9 +183,7 @@ public void shouldRefuseToUpgradeIfAllOfTheStoresWereNotShutDownCleanly() fileSystem.deleteRecursively( comparisonDirectory ); fileSystem.copyRecursively( dbDirectory, comparisonDirectory ); PageCache pageCache = pageCacheRule.getPageCache( fileSystem ); - UpgradableDatabase upgradableDatabase = new UpgradableDatabase( fileSystem, - new StoreVersionCheck( pageCache ), - getRecordFormats() ); + UpgradableDatabase upgradableDatabase = getUpgradableDatabase( pageCache ); try { @@ -206,9 +202,7 @@ public void shouldRefuseToUpgradeIfAllOfTheStoresWereNotShutDownCleanly() public void shouldContinueMovingFilesIfUpgradeCancelledWhileMoving() throws Exception { PageCache pageCache = pageCacheRule.getPageCache( fileSystem ); - UpgradableDatabase upgradableDatabase = new UpgradableDatabase( fileSystem, - new StoreVersionCheck( pageCache ), - getRecordFormats() ); + UpgradableDatabase upgradableDatabase = getUpgradableDatabase( pageCache ); String versionToMigrateTo = upgradableDatabase.currentVersion(); String versionToMigrateFrom = upgradableDatabase.checkUpgradeable( dbDirectory ).storeVersion(); @@ -257,9 +251,7 @@ public void upgradedNeoStoreShouldHaveNewUpgradeTimeAndUpgradeId() throws Except // Given fileSystem.deleteFile( new File( dbDirectory, INTERNAL_LOG_FILE ) ); PageCache pageCache = pageCacheRule.getPageCache( fileSystem ); - UpgradableDatabase upgradableDatabase = new UpgradableDatabase( fileSystem, - new StoreVersionCheck( pageCache ), - getRecordFormats() ); + UpgradableDatabase upgradableDatabase = getUpgradableDatabase( pageCache ); // When newUpgrader( upgradableDatabase, allowMigrateConfig, pageCache ).migrateIfNeeded( dbDirectory ); @@ -284,9 +276,7 @@ public void upgradeShouldNotLeaveLeftoverAndMigrationDirs() throws Exception // Given fileSystem.deleteFile( new File( dbDirectory, INTERNAL_LOG_FILE ) ); PageCache pageCache = pageCacheRule.getPageCache( fileSystem ); - UpgradableDatabase upgradableDatabase = new UpgradableDatabase( fileSystem, - new StoreVersionCheck( pageCache ), - getRecordFormats() ); + UpgradableDatabase upgradableDatabase = getUpgradableDatabase( pageCache ); // When newUpgrader( upgradableDatabase, allowMigrateConfig, pageCache ).migrateIfNeeded( dbDirectory ); @@ -300,8 +290,7 @@ public void upgradeShouldGiveProgressMonitorProgressMessages() throws Exception { // Given PageCache pageCache = pageCacheRule.getPageCache( fileSystem ); - UpgradableDatabase upgradableDatabase = new UpgradableDatabase( fileSystem, new StoreVersionCheck( pageCache ), - getRecordFormats() ); + UpgradableDatabase upgradableDatabase = getUpgradableDatabase( pageCache ); // When AssertableLogProvider logProvider = new AssertableLogProvider(); @@ -328,9 +317,7 @@ public void upgraderShouldCleanupLegacyLeftoverAndMigrationDirs() throws Excepti PageCache pageCache = pageCacheRule.getPageCache( fileSystem ); // When - UpgradableDatabase upgradableDatabase = new UpgradableDatabase( fileSystem, - new StoreVersionCheck( pageCache ), - getRecordFormats() ); + UpgradableDatabase upgradableDatabase = getUpgradableDatabase( pageCache ); StoreUpgrader storeUpgrader = newUpgrader( upgradableDatabase, pageCache ); storeUpgrader.migrateIfNeeded( dbDirectory ); @@ -344,6 +331,15 @@ protected void prepareSampleDatabase( String version, FileSystemAbstraction file prepareSampleLegacyDatabase( version, fileSystem, dbDirectory, databaseDirectory ); } + private UpgradableDatabase getUpgradableDatabase( PageCache pageCache ) + { + PhysicalLogFiles logFiles = new PhysicalLogFiles( dbDirectory, fileSystem ); + LogTailScanner tailScanner = new LogTailScanner( logFiles, fileSystem, new VersionAwareLogEntryReader<>() ); + return new UpgradableDatabase( + new StoreVersionCheck( pageCache ), + getRecordFormats(), tailScanner ); + } + private StoreMigrationParticipant participantThatWillFailWhenMoving( final String failureMessage ) { return new AbstractStoreMigrationParticipant( "Failing" ) @@ -391,11 +387,6 @@ private StoreUpgrader newUpgrader( UpgradableDatabase upgradableDatabase, PageCa return upgrader; } - private static void sneakyThrow( Throwable throwable ) throws T - { - throw (T) throwable; - } - private List migrationHelperDirs() { File[] tmpDirs = dbDirectory.listFiles( ( file, name ) -> file.isDirectory() && diff --git a/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/catchup/tx/TransactionLogCatchUpWriterTest.java b/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/catchup/tx/TransactionLogCatchUpWriterTest.java index 4a9cf9c96325e..b89ce9a38b954 100644 --- a/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/catchup/tx/TransactionLogCatchUpWriterTest.java +++ b/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/catchup/tx/TransactionLogCatchUpWriterTest.java @@ -111,7 +111,6 @@ public void shouldCreateTransactionLogWithCheckpoint() throws Exception // then verifyTransactionsInLog( fromTxId, endTxId ); verifyCheckpointInLog(); // necessary for recovery - } private void verifyCheckpointInLog() throws IOException @@ -122,9 +121,9 @@ private void verifyCheckpointInLog() throws IOException final LogTailScanner logTailScanner = new LogTailScanner( logFiles, fs, logEntryReader ); - LogTailScanner.LogTailInformation checkPoint = logTailScanner.find( 0 ); - assertNotNull( checkPoint.lastCheckPoint ); - assertTrue( checkPoint.commitsAfterLastCheckPoint ); + LogTailScanner.LogTailInformation tailInformation = logTailScanner.getTailInformation(); + assertNotNull( tailInformation.lastCheckPoint ); + assertTrue( tailInformation.commitsAfterLastCheckPoint ); } private void verifyTransactionsInLog( long fromTxId, long endTxId ) throws IOException diff --git a/enterprise/kernel/src/test/java/org/neo4j/kernel/impl/storemigration/StoreMigrationIT.java b/enterprise/kernel/src/test/java/org/neo4j/kernel/impl/storemigration/StoreMigrationIT.java index 157ee8732be5e..6ff4778173380 100644 --- a/enterprise/kernel/src/test/java/org/neo4j/kernel/impl/storemigration/StoreMigrationIT.java +++ b/enterprise/kernel/src/test/java/org/neo4j/kernel/impl/storemigration/StoreMigrationIT.java @@ -54,6 +54,9 @@ import org.neo4j.kernel.impl.store.format.standard.StandardV2_3; import org.neo4j.kernel.impl.store.format.standard.StandardV3_0; import org.neo4j.kernel.impl.store.format.standard.StandardV3_2; +import org.neo4j.kernel.impl.transaction.log.LogTailScanner; +import org.neo4j.kernel.impl.transaction.log.PhysicalLogFiles; +import org.neo4j.kernel.impl.transaction.log.entry.VersionAwareLogEntryReader; import org.neo4j.logging.NullLogProvider; import org.neo4j.test.rule.PageCacheRule; import org.neo4j.test.rule.TestDirectory; @@ -101,12 +104,14 @@ public static Iterable data() throws IOException PageCache pageCache = pageCacheRule.getPageCache( fs ); File dir = TestDirectory.testDirectory( StoreMigrationIT.class ).prepareDirectoryForTest( "migration" ); StoreVersionCheck storeVersionCheck = new StoreVersionCheck( pageCache ); + PhysicalLogFiles logFiles = new PhysicalLogFiles( dir, fs ); + LogTailScanner tailScanner = new LogTailScanner( logFiles, fs, new VersionAwareLogEntryReader<>() ); List data = new ArrayList<>(); ArrayList recordFormatses = new ArrayList<>(); RecordFormatSelector.allFormats().forEach( ( f ) -> addIfNotThere( f, recordFormatses ) ); for ( RecordFormats toFormat : recordFormatses ) { - UpgradableDatabase upgradableDatabase = new UpgradableDatabase( fs, storeVersionCheck, toFormat ); + UpgradableDatabase upgradableDatabase = new UpgradableDatabase( storeVersionCheck, toFormat, tailScanner ); for ( RecordFormats fromFormat : recordFormatses ) { File db = new File( dir, baseDirName( toFormat, fromFormat ) ); diff --git a/enterprise/kernel/src/test/java/org/neo4j/kernel/impl/storemigration/UpgradableDatabaseTest.java b/enterprise/kernel/src/test/java/org/neo4j/kernel/impl/storemigration/UpgradableDatabaseTest.java index 53db8ff18674f..2ffb745e8578e 100644 --- a/enterprise/kernel/src/test/java/org/neo4j/kernel/impl/storemigration/UpgradableDatabaseTest.java +++ b/enterprise/kernel/src/test/java/org/neo4j/kernel/impl/storemigration/UpgradableDatabaseTest.java @@ -43,6 +43,9 @@ import org.neo4j.kernel.impl.store.format.standard.Standard; import org.neo4j.kernel.impl.store.format.standard.StandardFormatFamily; import org.neo4j.kernel.impl.store.format.standard.StandardV2_3; +import org.neo4j.kernel.impl.transaction.log.LogTailScanner; +import org.neo4j.kernel.impl.transaction.log.PhysicalLogFiles; +import org.neo4j.kernel.impl.transaction.log.entry.VersionAwareLogEntryReader; import org.neo4j.kernel.internal.Version; import org.neo4j.test.rule.PageCacheRule; import org.neo4j.test.rule.TestDirectory; @@ -75,6 +78,7 @@ public static class SupportedVersions private File workingDirectory; private FileSystemAbstraction fileSystem; + private LogTailScanner tailScanner; @Parameterized.Parameter( 0 ) public String version; @@ -93,6 +97,8 @@ public void setup() throws IOException fileSystem = fileSystemRule.get(); workingDirectory = testDirectory.graphDbDir(); MigrationTestUtils.findFormatStoreDirectoryForVersion( version, workingDirectory ); + PhysicalLogFiles logFiles = new PhysicalLogFiles( workingDirectory, fileSystem ); + tailScanner = new LogTailScanner( logFiles, fileSystem, new VersionAwareLogEntryReader<>() ); } boolean storeFilesUpgradeable( File storeDirectory, UpgradableDatabase upgradableDatabase ) @@ -112,8 +118,8 @@ boolean storeFilesUpgradeable( File storeDirectory, UpgradableDatabase upgradabl public void shouldAcceptTheStoresInTheSampleDatabaseAsBeingEligibleForUpgrade() { // given - final UpgradableDatabase upgradableDatabase = new UpgradableDatabase( fileSystem, - new StoreVersionCheck( pageCacheRule.getPageCache( fileSystem ) ), getRecordFormat() ); + final UpgradableDatabase upgradableDatabase = new UpgradableDatabase( + new StoreVersionCheck( pageCacheRule.getPageCache( fileSystem ) ), getRecordFormat(), tailScanner ); // when final boolean result = storeFilesUpgradeable( workingDirectory, upgradableDatabase ); @@ -126,8 +132,8 @@ public void shouldAcceptTheStoresInTheSampleDatabaseAsBeingEligibleForUpgrade() public void shouldDetectOldVersionAsDifferentFromCurrent() throws Exception { // given - final UpgradableDatabase upgradableDatabase = new UpgradableDatabase( fileSystem, - new StoreVersionCheck( pageCacheRule.getPageCache( fileSystem ) ), getRecordFormat() ); + final UpgradableDatabase upgradableDatabase = new UpgradableDatabase( + new StoreVersionCheck( pageCacheRule.getPageCache( fileSystem ) ), getRecordFormat(), tailScanner ); // when boolean currentVersion = upgradableDatabase.hasCurrentVersion( workingDirectory ); @@ -144,8 +150,8 @@ public void shouldRejectStoresIfDBIsNotShutdownCleanly() throws IOException // given removeCheckPointFromTxLog( fileSystem, workingDirectory ); - final UpgradableDatabase upgradableDatabase = new UpgradableDatabase( fileSystem, - new StoreVersionCheck( pageCacheRule.getPageCache( fileSystem ) ), getRecordFormat() ); + final UpgradableDatabase upgradableDatabase = new UpgradableDatabase( + new StoreVersionCheck( pageCacheRule.getPageCache( fileSystem ) ), getRecordFormat(), tailScanner ); // when final boolean result = storeFilesUpgradeable( workingDirectory, upgradableDatabase ); @@ -170,6 +176,7 @@ public static class UnsupportedVersions private File workingDirectory; private FileSystemAbstraction fileSystem; + private LogTailScanner tailScanner; @Parameterized.Parameter( 0 ) public String version; @@ -191,14 +198,16 @@ public void setup() throws IOException File metadataStore = new File( workingDirectory, MetaDataStore.DEFAULT_NAME ); MetaDataStore.setRecord( pageCacheRule.getPageCache( fileSystem ), metadataStore, STORE_VERSION, MetaDataStore.versionStringToLong( version ) ); + PhysicalLogFiles logFiles = new PhysicalLogFiles( workingDirectory, fileSystem ); + tailScanner = new LogTailScanner( logFiles, fileSystem, new VersionAwareLogEntryReader<>() ); } @Test public void shouldDetectOldVersionAsDifferentFromCurrent() throws Exception { // given - final UpgradableDatabase upgradableDatabase = new UpgradableDatabase( fileSystem, - new StoreVersionCheck( pageCacheRule.getPageCache( fileSystem ) ), getRecordFormat() ); + final UpgradableDatabase upgradableDatabase = new UpgradableDatabase( + new StoreVersionCheck( pageCacheRule.getPageCache( fileSystem ) ), getRecordFormat(), tailScanner ); // when boolean currentVersion = upgradableDatabase.hasCurrentVersion( workingDirectory ); @@ -211,8 +220,8 @@ public void shouldDetectOldVersionAsDifferentFromCurrent() throws Exception public void shouldCommunicateWhatCausesInabilityToUpgrade() { // given - final UpgradableDatabase upgradableDatabase = new UpgradableDatabase( fileSystem, - new StoreVersionCheck( pageCacheRule.getPageCache( fileSystem ) ), getRecordFormat() ); + final UpgradableDatabase upgradableDatabase = new UpgradableDatabase( + new StoreVersionCheck( pageCacheRule.getPageCache( fileSystem ) ), getRecordFormat(), tailScanner ); try { // when diff --git a/tools/src/main/java/org/neo4j/tools/migration/StoreMigration.java b/tools/src/main/java/org/neo4j/tools/migration/StoreMigration.java index 3ae30aac85b2b..ad8ed34ca1605 100644 --- a/tools/src/main/java/org/neo4j/tools/migration/StoreMigration.java +++ b/tools/src/main/java/org/neo4j/tools/migration/StoreMigration.java @@ -46,9 +46,23 @@ import org.neo4j.kernel.impl.storemigration.StoreUpgrader; import org.neo4j.kernel.impl.storemigration.monitoring.VisibleMigrationProgressMonitor; import org.neo4j.kernel.impl.storemigration.participant.StoreMigrator; +<<<<<<< HEAD import org.neo4j.kernel.impl.transaction.state.DefaultSchemaIndexProviderMap; +======= +import org.neo4j.kernel.impl.transaction.log.FlushablePositionAwareChannel; +import org.neo4j.kernel.impl.transaction.log.LogHeaderCache; +import org.neo4j.kernel.impl.transaction.log.LogTailScanner; +import org.neo4j.kernel.impl.transaction.log.PhysicalLogFile; +import org.neo4j.kernel.impl.transaction.log.PhysicalLogFiles; +import org.neo4j.kernel.impl.transaction.log.ReadOnlyLogVersionRepository; +import org.neo4j.kernel.impl.transaction.log.ReadOnlyTransactionIdStore; +import org.neo4j.kernel.impl.transaction.log.TransactionLogWriter; +import org.neo4j.kernel.impl.transaction.log.entry.LogEntryWriter; +import org.neo4j.kernel.impl.transaction.log.entry.VersionAwareLogEntryReader; +>>>>>>> LogTailScanner updated import org.neo4j.kernel.impl.util.Dependencies; import org.neo4j.kernel.lifecycle.LifeSupport; +import org.neo4j.kernel.lifecycle.Lifespan; import org.neo4j.kernel.monitoring.Monitors; import org.neo4j.kernel.spi.legacyindex.IndexImplementation; import org.neo4j.kernel.spi.legacyindex.IndexProviders; @@ -120,6 +134,9 @@ public void run( final FileSystemAbstraction fs, final File storeDirectory, Conf kernelContext, GraphDatabaseDependencies.newDependencies().kernelExtensions(), deps, ignore() ) ); + final PhysicalLogFiles logFiles = new PhysicalLogFiles( storeDirectory, PhysicalLogFile.DEFAULT_NAME, fs ); + LogTailScanner tailScanner = new LogTailScanner( logFiles, fs, new VersionAwareLogEntryReader<>() ); + // Add the kernel store migrator life.start(); @@ -132,8 +149,12 @@ public void run( final FileSystemAbstraction fs, final File storeDirectory, Conf long startTime = System.currentTimeMillis(); DatabaseMigrator migrator = new DatabaseMigrator( progressMonitor, fs, config, logService, schemaIndexProviderMap, legacyIndexProvider.getIndexProviders(), - pageCache, RecordFormatSelector.selectForConfig( config, userLogProvider ) ); + pageCache, RecordFormatSelector.selectForConfig( config, userLogProvider ), tailScanner ); migrator.migrate( storeDirectory ); + + // Append checkpoint so the last log entry will have the latest version + appendCheckpoint( fs, storeDirectory, pageCache, logFiles, tailScanner ); + long duration = System.currentTimeMillis() - startTime; log.info( format( "Migration completed in %d s%n", duration / 1000 ) ); } @@ -147,9 +168,28 @@ public void run( final FileSystemAbstraction fs, final File storeDirectory, Conf } } - private class LegacyIndexProvider implements IndexProviders + private void appendCheckpoint( FileSystemAbstraction fs, File storeDirectory, PageCache pageCache, + PhysicalLogFiles logFiles, LogTailScanner tailScanner ) throws IOException { + ReadOnlyLogVersionRepository logVersionRepository = new ReadOnlyLogVersionRepository( pageCache, storeDirectory ); + ReadOnlyTransactionIdStore + readOnlyTransactionIdStore = new ReadOnlyTransactionIdStore( pageCache, storeDirectory ); + PhysicalLogFile logFile = new PhysicalLogFile( fs, logFiles, Long.MAX_VALUE /*don't rotate*/, + () -> readOnlyTransactionIdStore.getLastClosedTransactionId() - 1, logVersionRepository, + PhysicalLogFile.NO_MONITOR, + new LogHeaderCache( 10 ) ); + + try ( Lifespan lifespan = new Lifespan( logFile ) ) + { + FlushablePositionAwareChannel writer = logFile.getWriter(); + TransactionLogWriter transactionLogWriter = new TransactionLogWriter( new LogEntryWriter( writer ) ); + transactionLogWriter.checkPoint( tailScanner.getTailInformation().lastCheckPoint.getLogPosition() ); + writer.prepareForFlush().flush(); + } + } + private class LegacyIndexProvider implements IndexProviders + { private final Map indexProviders = new HashMap<>(); public Map getIndexProviders()