From 185ecaa585f82acd069fd9c29f4bf0309ded1994 Mon Sep 17 00:00:00 2001 From: MishaDemianenko Date: Tue, 21 Nov 2017 13:49:32 +0100 Subject: [PATCH] Allow transaction logs to be located in a separate folder. Previously transaction logs were always located in the same directory as a store. This PR will allow changing that by specifying "logical_logs_location" with a path to a folder where transaction logs should be stored. This will allow placing transaction logs on a separate dedicated drive. CC integration tests will always put transaction logs in separate directories. HA should be able to copy store from another instance with custom transactional logs directory and place them into the correct directory for that specific instance. Backup will put transaction logs together with store files and restore command will properly put them into configured transaction logs folder --- .../main/java/org/neo4j/commandline/Util.java | 14 ++ .../consistency/CheckConsistencyCommand.java | 7 +- .../consistency/ConsistencyCheckTool.java | 5 +- .../consistency/ConsistencyCheckToolTest.java | 56 +++++-- .../neo4j/commandline/dbms/CsvImporter.java | 4 +- .../commandline/dbms/DatabaseImporter.java | 2 +- .../neo4j/commandline/dbms/DumpCommand.java | 33 ++-- .../neo4j/commandline/dbms/ImportCommand.java | 6 +- .../neo4j/commandline/dbms/LoadCommand.java | 41 +++-- .../DatabaseManagementSystemSettings.java | 20 +-- .../java/org/neo4j/dbms/archive/Dumper.java | 36 +++-- .../java/org/neo4j/dbms/archive/Loader.java | 52 +++++-- .../commandline/dbms/CsvImporterTest.java | 5 +- .../dbms/DatabaseImporterTest.java | 3 +- .../commandline/dbms/DumpCommandTest.java | 64 +++++--- .../commandline/dbms/LoadCommandTest.java | 55 ++++--- .../org/neo4j/commandline/dbms/UtilTest.java | 34 +++++ .../DatabaseManagementSystemSettingsTest.java | 5 +- .../org/neo4j/dbms/archive/ArchiveTest.java | 39 ++++- .../org/neo4j/dbms/archive/DumperTest.java | 19 ++- .../org/neo4j/dbms/archive/LoaderTest.java | 82 ++++++++-- .../io/fs/DefaultFileSystemAbstraction.java | 6 + .../io/fs/DelegateFileSystemAbstraction.java | 6 + .../neo4j/io/fs/FileSystemAbstraction.java | 2 + .../main/java/org/neo4j/io/fs/FileUtils.java | 42 ++++-- .../fs/AdversarialFileSystemAbstraction.java | 9 ++ .../DelegatingFileSystemAbstraction.java | 6 + .../EphemeralFileSystemAbstraction.java | 8 + .../SelectiveFileSystemAbstraction.java | 6 + .../io/fs/FileSystemAbstractionTest.java | 46 ++++-- .../neo4j/test/rule/fs/FileSystemRule.java | 6 + .../factory/GraphDatabaseSettings.java | 17 +++ .../org/neo4j/kernel/NeoStoreDataSource.java | 27 ++-- .../neo4j/kernel/configuration/Settings.java | 142 ++++++++++-------- .../kernel/impl/factory/PlatformModule.java | 4 - .../recovery/RecoveryRequiredChecker.java | 9 +- .../participant/StoreMigrator.java | 6 +- .../log/ReadOnlyTransactionStore.java | 6 +- .../impl/transaction/log/files/LogFiles.java | 4 + .../log/files/LogFilesBuilder.java | 47 +++++- .../log/files/TransactionLogFiles.java | 26 +++- .../log/files/TransactionLogFilesHelper.java | 2 +- .../state/NeoStoreFileListing.java | 32 ++-- .../storageengine/api/StoreFileMetadata.java | 12 ++ .../TransactionLogsInSeparateLocationIT.java | 100 ++++++++++++ .../recovery/RecoveryRequiredCheckerTest.java | 74 +++++++-- .../kernel/impl/store/TestStoreAccess.java | 4 +- .../participant/StoreMigratorTest.java | 53 +++++++ .../log/files/LogFilesBuilderTest.java | 60 ++++++-- .../log/files/TransactionLogFilesTest.java | 43 ++++-- .../log/stresstest/workload/Runner.java | 4 +- .../state/NeoStoreFileListingTest.java | 64 ++++++-- ...ersarialPageCacheGraphDatabaseFactory.java | 3 +- .../neo4j/test/TestGraphDatabaseFactory.java | 1 + .../AbstractInProcessServerBuilder.java | 2 +- .../neo4j/harness/InProcessBuilderTestIT.java | 5 +- .../org/neo4j/server/CommunityNeoServer.java | 4 +- .../database/LifecycleManagingDatabase.java | 4 +- .../neo4j/server/BaseBootstrapperTestIT.java | 2 +- .../neo4j/server/ServerBootstrapperTest.java | 4 +- .../org/neo4j/server/ServerTestUtils.java | 3 +- .../configuration/ConfigFileBuilder.java | 4 +- .../configuration/ConfigLoaderTest.java | 6 +- .../TestLifecycleManagedDatabase.java | 6 +- .../helpers/CommunityServerBuilder.java | 7 +- .../server/integration/StartupLoggingIT.java | 9 +- ...bstractBackupSupportingClassesFactory.java | 7 +- .../org/neo4j/backup/BackupDelegator.java | 2 +- .../neo4j/backup/BackupProtocolService.java | 2 - .../neo4j/backup/BackupRecoveryService.java | 2 + .../neo4j/backup/BackupStrategyWrapper.java | 4 - .../OnlineBackupCommandConfigLoader.java | 5 +- .../backup/OnlineBackupRequiredArguments.java | 3 +- .../org/neo4j/restore/RestoreDatabaseCli.java | 4 +- .../neo4j/restore/RestoreDatabaseCommand.java | 27 +++- .../org/neo4j/backup/BackupDelegatorTest.java | 2 +- .../java/org/neo4j/backup/BackupFlowTest.java | 23 +-- .../test/java/org/neo4j/backup/BackupIT.java | 40 ++++- .../neo4j/backup/BackupProtocolServiceIT.java | 2 +- .../OnlineBackupCommandConfigLoaderTest.java | 2 - .../neo4j/causalclustering/BackupCoreIT.java | 9 +- .../causalclustering/ClusterSeedingIT.java | 9 +- .../restore/RestoreDatabaseCommandIT.java | 96 ++++++++++-- .../storecopy/CopiedStoreRecovery.java | 4 +- .../catchup/storecopy/LocalDatabase.java | 20 ++- .../catchup/storecopy/RemoteStore.java | 24 ++- .../catchup/storecopy/StoreCopyProcess.java | 4 +- .../catchup/storecopy/StoreFiles.java | 18 ++- .../storecopy/TemporaryStoreDirectory.java | 8 +- .../tx/TransactionLogCatchUpFactory.java | 7 +- .../tx/TransactionLogCatchUpWriter.java | 14 +- .../core/EnterpriseCoreEditionModule.java | 22 ++- .../core/server/CoreServerModule.java | 6 +- .../core/state/CoreBootstrapper.java | 1 + .../state/snapshot/CoreStateDownloader.java | 3 +- .../EnterpriseReadReplicaEditionModule.java | 27 +++- .../dbms/UnbindFromClusterCommand.java | 8 +- .../backup/RestoreClusterUtils.java | 18 ++- .../storecopy/CopiedStoreRecoveryTest.java | 3 +- .../catchup/storecopy/LocalDatabaseTest.java | 3 +- .../catchup/storecopy/RemoteStoreTest.java | 11 +- .../catchup/storecopy/StoreFilesTest.java | 18 ++- .../tx/TransactionLogCatchUpWriterTest.java | 59 ++++++-- .../core/state/CoreBootstrapperIT.java | 40 ++++- .../id/RebuildReplicatedIdGeneratorsTest.java | 1 - .../snapshot/CoreStateDownloaderTest.java | 14 +- .../discovery/CoreClusterMember.java | 11 +- .../discovery/ReadReplica.java | 2 + .../scenarios/ClusterCustomLogLocationIT.java | 99 ++++++++++++ .../neo4j/com/storecopy/FileMoveAction.java | 46 ++++-- .../neo4j/com/storecopy/MoveAfterCopy.java | 9 +- .../neo4j/com/storecopy/StoreCopyClient.java | 9 +- .../neo4j/com/storecopy/StoreCopyServer.java | 13 +- .../com/storecopy/StoreCopyClientTest.java | 133 +++++++++++----- .../fulltext/integrations/bloom/BloomIT.java | 62 +------- .../SwitchToSlaveCopyThenBranchTest.java | 3 +- .../test/java/org/neo4j/util/TestHelpers.java | 6 - .../enterprise/EnterpriseNeoServer.java | 12 +- .../EnterpriseBootstrapperTestIT.java | 2 +- .../enterprise/jmx/ServerManagementIT.java | 7 +- .../neo4j/storeupgrade/StoreUpgradeIT.java | 9 +- .../applytx/ApplyTransactionsCommand.java | 14 +- .../neo4j/tools/migration/StoreMigration.java | 3 +- 123 files changed, 1841 insertions(+), 686 deletions(-) create mode 100644 community/kernel/src/test/java/org/neo4j/graphdb/TransactionLogsInSeparateLocationIT.java create mode 100644 enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/scenarios/ClusterCustomLogLocationIT.java diff --git a/community/command-line/src/main/java/org/neo4j/commandline/Util.java b/community/command-line/src/main/java/org/neo4j/commandline/Util.java index cbbaccfa0c2dc..d724e529b0ed5 100644 --- a/community/command-line/src/main/java/org/neo4j/commandline/Util.java +++ b/community/command-line/src/main/java/org/neo4j/commandline/Util.java @@ -63,6 +63,20 @@ public static Path canonicalPath( File file ) throws IllegalArgumentException } } + public static boolean isSameOrChildFile( File parent, File candidate ) + { + Path canonicalCandidate = canonicalPath( candidate ); + Path canonicalParentPath = canonicalPath( parent ); + return canonicalCandidate.startsWith( canonicalParentPath ); + } + + public static boolean isSameOrChildPath( Path parent, Path candidate ) + { + Path normalizedCandidate = candidate.normalize(); + Path normalizedParent = parent.normalize(); + return normalizedCandidate.startsWith( normalizedParent ); + } + public static void checkLock( Path databaseDirectory ) throws CommandFailed { try ( FileSystemAbstraction fileSystem = new DefaultFileSystemAbstraction(); diff --git a/community/consistency-check/src/main/java/org/neo4j/consistency/CheckConsistencyCommand.java b/community/consistency-check/src/main/java/org/neo4j/consistency/CheckConsistencyCommand.java index 5b63b6cf2d5c1..9255fd0d076df 100644 --- a/community/consistency-check/src/main/java/org/neo4j/consistency/CheckConsistencyCommand.java +++ b/community/consistency-check/src/main/java/org/neo4j/consistency/CheckConsistencyCommand.java @@ -36,7 +36,6 @@ import org.neo4j.commandline.arguments.common.OptionalCanonicalPath; import org.neo4j.consistency.checking.full.ConsistencyCheckIncompleteException; import org.neo4j.consistency.checking.full.ConsistencyFlags; -import org.neo4j.dbms.DatabaseManagementSystemSettings; import org.neo4j.graphdb.factory.GraphDatabaseSettings; import org.neo4j.helpers.Strings; import org.neo4j.helpers.collection.MapUtil; @@ -51,7 +50,7 @@ import org.neo4j.logging.FormattedLogProvider; import static java.lang.String.format; -import static org.neo4j.dbms.DatabaseManagementSystemSettings.database_path; +import static org.neo4j.graphdb.factory.GraphDatabaseSettings.database_path; public class CheckConsistencyCommand implements AdminCommand { @@ -234,7 +233,7 @@ private void checkDbState( File storeDir, Config additionalConfiguration ) throw .createPageCache( fileSystem, additionalConfiguration ) ) { RecoveryRequiredChecker requiredChecker = - new RecoveryRequiredChecker( fileSystem, pageCache, new Monitors() ); + new RecoveryRequiredChecker( fileSystem, pageCache, additionalConfiguration, new Monitors() ); if ( requiredChecker.isRecoveryRequiredAt( storeDir ) ) { throw new CommandFailed( @@ -253,7 +252,7 @@ private void checkDbState( File storeDir, Config additionalConfiguration ) throw private static Config loadNeo4jConfig( Path homeDir, Path configDir, String databaseName, Map additionalConfig ) { - additionalConfig.put( DatabaseManagementSystemSettings.active_database.name(), databaseName ); + additionalConfig.put( GraphDatabaseSettings.active_database.name(), databaseName ); return Config.fromFile( configDir.resolve( Config.DEFAULT_CONFIG_FILE_NAME ) ).withHome( homeDir ).withConnectorsDisabled() .withSettings( additionalConfig ).build(); diff --git a/community/consistency-check/src/main/java/org/neo4j/consistency/ConsistencyCheckTool.java b/community/consistency-check/src/main/java/org/neo4j/consistency/ConsistencyCheckTool.java index e910ed152b4c9..16e9b21afb262 100644 --- a/community/consistency-check/src/main/java/org/neo4j/consistency/ConsistencyCheckTool.java +++ b/community/consistency-check/src/main/java/org/neo4j/consistency/ConsistencyCheckTool.java @@ -26,8 +26,8 @@ import java.util.List; import java.util.Map; -import org.neo4j.consistency.checking.full.ConsistencyFlags; import org.neo4j.consistency.checking.full.ConsistencyCheckIncompleteException; +import org.neo4j.consistency.checking.full.ConsistencyFlags; import org.neo4j.graphdb.factory.GraphDatabaseSettings; import org.neo4j.helpers.Args; import org.neo4j.helpers.Strings; @@ -137,7 +137,8 @@ private void checkDbState( File storeDir, Config tuningConfiguration ) throws To { try ( PageCache pageCache = ConfigurableStandalonePageCacheFactory.createPageCache( fs, tuningConfiguration ) ) { - RecoveryRequiredChecker requiredChecker = new RecoveryRequiredChecker( fs, pageCache, new Monitors() ); + RecoveryRequiredChecker requiredChecker = new RecoveryRequiredChecker( fs, pageCache, + tuningConfiguration, new Monitors() ); if ( requiredChecker.isRecoveryRequiredAt( storeDir ) ) { throw new ToolFailureException( Strings.joinAsLines( diff --git a/community/consistency-check/src/test/java/org/neo4j/consistency/ConsistencyCheckToolTest.java b/community/consistency-check/src/test/java/org/neo4j/consistency/ConsistencyCheckToolTest.java index ef9a1c9fa23b9..d3b672cb06859 100644 --- a/community/consistency-check/src/test/java/org/neo4j/consistency/ConsistencyCheckToolTest.java +++ b/community/consistency-check/src/test/java/org/neo4j/consistency/ConsistencyCheckToolTest.java @@ -43,6 +43,7 @@ import org.neo4j.graphdb.GraphDatabaseService; import org.neo4j.graphdb.Transaction; import org.neo4j.graphdb.factory.GraphDatabaseSettings; +import org.neo4j.helpers.collection.MapUtil; import org.neo4j.helpers.progress.ProgressMonitorFactory; import org.neo4j.io.fs.DefaultFileSystemAbstraction; import org.neo4j.io.fs.FileSystemAbstraction; @@ -66,20 +67,21 @@ import static org.mockito.Mockito.verify; import static org.mockito.Mockito.verifyZeroInteractions; import static org.neo4j.graphdb.Label.label; +import static org.neo4j.graphdb.factory.GraphDatabaseSettings.logical_logs_location; public class ConsistencyCheckToolTest { - private final TestDirectory storeDirectory = TestDirectory.testDirectory(); + private final TestDirectory testDirectory = TestDirectory.testDirectory(); private final EphemeralFileSystemRule fs = new EphemeralFileSystemRule(); @Rule - public RuleChain ruleChain = RuleChain.outerRule( storeDirectory ).around( fs ); + public RuleChain ruleChain = RuleChain.outerRule( testDirectory ).around( fs ); @Test public void runsConsistencyCheck() throws Exception { // given - File storeDir = storeDirectory.directory(); + File storeDir = testDirectory.directory(); String[] args = {storeDir.getPath()}; ConsistencyCheckService service = mock( ConsistencyCheckService.class ); @@ -108,8 +110,8 @@ public void consistencyCheckerLogUseSystemTimezoneIfConfigurable() throws Except provider.getLog( "test" ).info( "testMessage" ); return ConsistencyCheckService.Result.success( new File( StringUtils.EMPTY ) ); } ); - File storeDir = storeDirectory.directory(); - File configFile = storeDirectory.file( Config.DEFAULT_CONFIG_FILE_NAME ); + File storeDir = testDirectory.directory(); + File configFile = testDirectory.file( Config.DEFAULT_CONFIG_FILE_NAME ); Properties properties = new Properties(); properties.setProperty( GraphDatabaseSettings.log_timezone.name(), LogTimeZone.SYSTEM.name() ); properties.store( new FileWriter( configFile ), null ); @@ -128,7 +130,7 @@ public void consistencyCheckerLogUseSystemTimezoneIfConfigurable() throws Except public void appliesDefaultTuningConfigurationForConsistencyChecker() throws Exception { // given - File storeDir = storeDirectory.directory(); + File storeDir = testDirectory.directory(); String[] args = {storeDir.getPath()}; ConsistencyCheckService service = mock( ConsistencyCheckService.class ); @@ -147,8 +149,8 @@ public void appliesDefaultTuningConfigurationForConsistencyChecker() throws Exce public void passesOnConfigurationIfProvided() throws Exception { // given - File storeDir = storeDirectory.directory(); - File configFile = storeDirectory.file( Config.DEFAULT_CONFIG_FILE_NAME ); + File storeDir = testDirectory.directory(); + File configFile = testDirectory.file( Config.DEFAULT_CONFIG_FILE_NAME ); Properties properties = new Properties(); properties.setProperty( ConsistencyCheckSettings.consistency_check_property_owners.name(), "true" ); properties.store( new FileWriter( configFile ), null ); @@ -191,8 +193,8 @@ public void exitWithFailureIndicatingCorrectUsageIfNoArgumentsSupplied() throws public void exitWithFailureIfConfigSpecifiedButConfigFileDoesNotExist() throws Exception { // given - File configFile = storeDirectory.file( "nonexistent_file" ); - String[] args = {storeDirectory.directory().getPath(), "-config", configFile.getPath()}; + File configFile = testDirectory.file( "nonexistent_file" ); + String[] args = {testDirectory.directory().getPath(), "-config", configFile.getPath()}; ConsistencyCheckService service = mock( ConsistencyCheckService.class ); try @@ -214,9 +216,34 @@ public void exitWithFailureIfConfigSpecifiedButConfigFileDoesNotExist() throws E @Test( expected = ToolFailureException.class ) public void failWhenStoreWasNonCleanlyShutdown() throws Exception { - createGraphDbAndKillIt(); + createGraphDbAndKillIt( Config.defaults() ); - runConsistencyCheckToolWith( fs.get(), storeDirectory.graphDbDir().getAbsolutePath() ); + runConsistencyCheckToolWith( fs.get(), testDirectory.graphDbDir().getAbsolutePath() ); + } + + @Test( expected = ToolFailureException.class ) + public void failOnNotCleanlyShutdownStoreWithLogsInCustomRelativeLocation() throws Exception + { + File customConfigFile = testDirectory.file( "customConfig" ); + Config customConfig = Config.defaults( logical_logs_location, "otherLocation" ); + createGraphDbAndKillIt( customConfig ); + MapUtil.store( customConfig.getRaw(), customConfigFile ); + String[] args = {testDirectory.graphDbDir().getPath(), "-config", customConfigFile.getPath()}; + + runConsistencyCheckToolWith( fs.get(), args ); + } + + @Test( expected = ToolFailureException.class ) + public void failOnNotCleanlyShutdownStoreWithLogsInCustomAbsoluteLocation() throws Exception + { + File customConfigFile = testDirectory.file( "customConfig" ); + File otherLocation = testDirectory.directory( "otherLocation" ); + Config customConfig = Config.defaults( logical_logs_location, otherLocation.getAbsolutePath() ); + createGraphDbAndKillIt( customConfig ); + MapUtil.store( customConfig.getRaw(), customConfigFile ); + String[] args = {testDirectory.graphDbDir().getPath(), "-config", customConfigFile.getPath()}; + + runConsistencyCheckToolWith( fs.get(), args ); } private void checkLogRecordTimeZone( ConsistencyCheckService service, String[] args, int hoursShift, @@ -237,11 +264,12 @@ private String readLogLine( ByteArrayOutputStream outputStream ) throws IOExcept return bufferedReader.readLine(); } - private void createGraphDbAndKillIt() throws Exception + private void createGraphDbAndKillIt( Config config ) throws Exception { final GraphDatabaseService db = new TestGraphDatabaseFactory() .setFileSystem( fs.get() ) - .newImpermanentDatabaseBuilder( storeDirectory.graphDbDir() ) + .newImpermanentDatabaseBuilder( testDirectory.graphDbDir() ) + .setConfig( config.getRaw() ) .newGraphDatabase(); try ( Transaction tx = db.beginTx() ) diff --git a/community/dbms/src/main/java/org/neo4j/commandline/dbms/CsvImporter.java b/community/dbms/src/main/java/org/neo4j/commandline/dbms/CsvImporter.java index d30861ffae8f7..75974c7d1d9c7 100644 --- a/community/dbms/src/main/java/org/neo4j/commandline/dbms/CsvImporter.java +++ b/community/dbms/src/main/java/org/neo4j/commandline/dbms/CsvImporter.java @@ -30,7 +30,6 @@ import org.neo4j.commandline.admin.OutsideWorld; import org.neo4j.commandline.dbms.config.WrappedBatchImporterConfigurationForNeo4jAdmin; import org.neo4j.commandline.dbms.config.WrappedCsvInputConfigurationForNeo4jAdmin; -import org.neo4j.dbms.DatabaseManagementSystemSettings; import org.neo4j.graphdb.factory.GraphDatabaseSettings; import org.neo4j.helpers.Args; import org.neo4j.io.fs.FileSystemAbstraction; @@ -43,7 +42,6 @@ import org.neo4j.unsafe.impl.batchimport.input.csv.IdType; import static java.nio.charset.Charset.defaultCharset; - import static org.neo4j.kernel.impl.util.Converters.withDefault; import static org.neo4j.tooling.ImportTool.csvConfiguration; import static org.neo4j.tooling.ImportTool.extractInputFiles; @@ -100,7 +98,7 @@ class CsvImporter implements Importer public void doImport() throws IOException { FileSystemAbstraction fs = outsideWorld.fileSystem(); - File storeDir = databaseConfig.get( DatabaseManagementSystemSettings.database_path ); + File storeDir = databaseConfig.get( GraphDatabaseSettings.database_path ); File logsDir = databaseConfig.get( GraphDatabaseSettings.logs_directory ); File reportFile = new File( reportFileName ); diff --git a/community/dbms/src/main/java/org/neo4j/commandline/dbms/DatabaseImporter.java b/community/dbms/src/main/java/org/neo4j/commandline/dbms/DatabaseImporter.java index a539954720b2a..d126a3197df05 100644 --- a/community/dbms/src/main/java/org/neo4j/commandline/dbms/DatabaseImporter.java +++ b/community/dbms/src/main/java/org/neo4j/commandline/dbms/DatabaseImporter.java @@ -30,7 +30,7 @@ import org.neo4j.kernel.impl.util.Converters; import org.neo4j.kernel.impl.util.Validators; -import static org.neo4j.dbms.DatabaseManagementSystemSettings.database_path; +import static org.neo4j.graphdb.factory.GraphDatabaseSettings.database_path; class DatabaseImporter implements Importer { diff --git a/community/dbms/src/main/java/org/neo4j/commandline/dbms/DumpCommand.java b/community/dbms/src/main/java/org/neo4j/commandline/dbms/DumpCommand.java index a7d3e775e92b3..4549164726262 100644 --- a/community/dbms/src/main/java/org/neo4j/commandline/dbms/DumpCommand.java +++ b/community/dbms/src/main/java/org/neo4j/commandline/dbms/DumpCommand.java @@ -32,8 +32,8 @@ import org.neo4j.commandline.admin.CommandFailed; import org.neo4j.commandline.admin.IncorrectUsage; import org.neo4j.commandline.arguments.Arguments; -import org.neo4j.dbms.DatabaseManagementSystemSettings; import org.neo4j.dbms.archive.Dumper; +import org.neo4j.graphdb.factory.GraphDatabaseSettings; import org.neo4j.kernel.StoreLockException; import org.neo4j.kernel.configuration.Config; import org.neo4j.kernel.impl.util.Validators; @@ -41,7 +41,8 @@ import static java.lang.String.format; import static org.neo4j.commandline.Util.canonicalPath; -import static org.neo4j.dbms.DatabaseManagementSystemSettings.database_path; +import static org.neo4j.graphdb.factory.GraphDatabaseSettings.database_path; +import static org.neo4j.graphdb.factory.GraphDatabaseSettings.logical_logs_location; public class DumpCommand implements AdminCommand { @@ -66,7 +67,10 @@ public void execute( String[] args ) throws IncorrectUsage, CommandFailed { String database = arguments.parse( args ).get( "database" ); Path archive = calculateArchive( database, arguments.getMandatoryPath( "to" ) ); - Path databaseDirectory = canonicalPath( toDatabaseDirectory( database ) ); + + Config config = buildConfig( database ); + Path databaseDirectory = canonicalPath( getDatabaseDirectory( config ) ); + Path transactionLogsDirectory = canonicalPath( getTransactionalLogsDirectory( config ) ); try { @@ -79,7 +83,7 @@ public void execute( String[] args ) throws IncorrectUsage, CommandFailed try ( Closeable ignored = StoreLockChecker.check( databaseDirectory ) ) { - dump( database, databaseDirectory, archive ); + dump( database, databaseDirectory, transactionLogsDirectory, archive ); } catch ( StoreLockException e ) { @@ -97,13 +101,23 @@ public void execute( String[] args ) throws IncorrectUsage, CommandFailed } } - private Path toDatabaseDirectory( String databaseName ) + private Path getDatabaseDirectory( Config config ) + { + return config.get( database_path ).toPath(); + } + + private Path getTransactionalLogsDirectory( Config config ) + { + return config.get( logical_logs_location ).toPath(); + } + + private Config buildConfig( String databaseName ) { return Config.fromFile( configDir.resolve( Config.DEFAULT_CONFIG_FILE_NAME ) ) .withHome( homeDir ) .withConnectorsDisabled() - .withSetting( DatabaseManagementSystemSettings.active_database, databaseName ) - .build().get( database_path ).toPath(); + .withSetting( GraphDatabaseSettings.active_database, databaseName ) + .build(); } private Path calculateArchive( String database, Path to ) @@ -111,11 +125,12 @@ private Path calculateArchive( String database, Path to ) return Files.isDirectory( to ) ? to.resolve( database + ".dump" ) : to; } - private void dump( String database, Path databaseDirectory, Path archive ) throws CommandFailed + private void dump( String database, Path databaseDirectory, Path transactionalLogsDirectory, Path archive ) + throws CommandFailed { try { - dumper.dump( databaseDirectory, archive, this::isStoreLock ); + dumper.dump( databaseDirectory, transactionalLogsDirectory, archive, this::isStoreLock ); } catch ( FileAlreadyExistsException e ) { diff --git a/community/dbms/src/main/java/org/neo4j/commandline/dbms/ImportCommand.java b/community/dbms/src/main/java/org/neo4j/commandline/dbms/ImportCommand.java index 08a2d53181589..2eb17447511a2 100644 --- a/community/dbms/src/main/java/org/neo4j/commandline/dbms/ImportCommand.java +++ b/community/dbms/src/main/java/org/neo4j/commandline/dbms/ImportCommand.java @@ -34,7 +34,7 @@ import org.neo4j.commandline.arguments.OptionalBooleanArg; import org.neo4j.commandline.arguments.OptionalNamedArg; import org.neo4j.commandline.arguments.OptionalNamedArgWithMetadata; -import org.neo4j.dbms.DatabaseManagementSystemSettings; +import org.neo4j.graphdb.factory.GraphDatabaseSettings; import org.neo4j.helpers.Args; import org.neo4j.helpers.collection.MapUtil; import org.neo4j.kernel.configuration.Config; @@ -250,7 +250,7 @@ public void execute( String[] args ) throws IncorrectUsage, CommandFailed Config config = loadNeo4jConfig( homeDir, configDir, database, loadAdditionalConfig( additionalConfigFile ) ); Validators.CONTAINS_NO_EXISTING_DATABASE - .validate( config.get( DatabaseManagementSystemSettings.database_path ) ); + .validate( config.get( GraphDatabaseSettings.database_path ) ); Importer importer = importerFactory.getImporterForMode( mode, Args.parse( args ), config, outsideWorld ); importer.doImport(); @@ -288,7 +288,7 @@ private static Config loadNeo4jConfig( Path homeDir, Path configDir, String data { return Config.fromFile( configDir.resolve( Config.DEFAULT_CONFIG_FILE_NAME ) ) .withHome( homeDir ) - .withSetting( DatabaseManagementSystemSettings.active_database, databaseName ) + .withSetting( GraphDatabaseSettings.active_database, databaseName ) .withSettings( additionalConfig ) .withConnectorsDisabled().build(); } diff --git a/community/dbms/src/main/java/org/neo4j/commandline/dbms/LoadCommand.java b/community/dbms/src/main/java/org/neo4j/commandline/dbms/LoadCommand.java index 8538c6fdc36a7..914e0aee7f6f4 100644 --- a/community/dbms/src/main/java/org/neo4j/commandline/dbms/LoadCommand.java +++ b/community/dbms/src/main/java/org/neo4j/commandline/dbms/LoadCommand.java @@ -32,17 +32,19 @@ import org.neo4j.commandline.arguments.Arguments; import org.neo4j.commandline.arguments.OptionalBooleanArg; import org.neo4j.commandline.arguments.common.MandatoryCanonicalPath; -import org.neo4j.dbms.DatabaseManagementSystemSettings; import org.neo4j.dbms.archive.IncorrectFormat; import org.neo4j.dbms.archive.Loader; +import org.neo4j.graphdb.factory.GraphDatabaseSettings; import org.neo4j.io.fs.FileUtils; import org.neo4j.kernel.configuration.Config; import static java.util.Objects.requireNonNull; import static org.neo4j.commandline.Util.canonicalPath; import static org.neo4j.commandline.Util.checkLock; +import static org.neo4j.commandline.Util.isSameOrChildPath; import static org.neo4j.commandline.Util.wrapIOException; -import static org.neo4j.dbms.DatabaseManagementSystemSettings.database_path; +import static org.neo4j.graphdb.factory.GraphDatabaseSettings.database_path; +import static org.neo4j.graphdb.factory.GraphDatabaseSettings.logical_logs_location; public class LoadCommand implements AdminCommand { @@ -74,22 +76,35 @@ public void execute( String[] args ) throws IncorrectUsage, CommandFailed String database = arguments.get( "database" ); boolean force = arguments.getBoolean( "force" ); - Path databaseDirectory = canonicalPath( toDatabaseDirectory( database ) ); + Config config = buildConfig( database ); - deleteIfNecessary( databaseDirectory, force ); - load( archive, database, databaseDirectory ); + Path databaseDirectory = canonicalPath( getDatabaseDirectory( config ) ); + Path transactionLogsDirectory = canonicalPath( getTransactionalLogsDirectory( config ) ); + + deleteIfNecessary( databaseDirectory, transactionLogsDirectory, force ); + load( archive, database, databaseDirectory, transactionLogsDirectory ); + } + + private Path getDatabaseDirectory( Config config ) + { + return config.get( database_path ).toPath(); + } + + private Path getTransactionalLogsDirectory( Config config ) + { + return config.get( logical_logs_location ).toPath(); } - private Path toDatabaseDirectory( String databaseName ) + private Config buildConfig( String databaseName ) { return Config.fromFile( configDir.resolve( Config.DEFAULT_CONFIG_FILE_NAME ) ) .withHome( homeDir ) .withConnectorsDisabled() - .withSetting( DatabaseManagementSystemSettings.active_database, databaseName ) - .build().get( database_path ).toPath(); + .withSetting( GraphDatabaseSettings.active_database, databaseName ) + .build(); } - private void deleteIfNecessary( Path databaseDirectory, boolean force ) throws CommandFailed + private void deleteIfNecessary( Path databaseDirectory, Path transactionLogsDirectory, boolean force ) throws CommandFailed { try { @@ -97,6 +112,10 @@ private void deleteIfNecessary( Path databaseDirectory, boolean force ) throws C { checkLock( databaseDirectory ); FileUtils.deletePathRecursively( databaseDirectory ); + if ( !isSameOrChildPath( databaseDirectory, transactionLogsDirectory ) ) + { + FileUtils.deletePathRecursively( transactionLogsDirectory ); + } } } catch ( IOException e ) @@ -105,11 +124,11 @@ private void deleteIfNecessary( Path databaseDirectory, boolean force ) throws C } } - private void load( Path archive, String database, Path databaseDirectory ) throws CommandFailed + private void load( Path archive, String database, Path databaseDirectory, Path transactionLogsDirectory ) throws CommandFailed { try { - loader.load( archive, databaseDirectory ); + loader.load( archive, databaseDirectory, transactionLogsDirectory ); } catch ( NoSuchFileException e ) { diff --git a/community/dbms/src/main/java/org/neo4j/dbms/DatabaseManagementSystemSettings.java b/community/dbms/src/main/java/org/neo4j/dbms/DatabaseManagementSystemSettings.java index 74c478c9e951c..37ceb2bdbde1f 100644 --- a/community/dbms/src/main/java/org/neo4j/dbms/DatabaseManagementSystemSettings.java +++ b/community/dbms/src/main/java/org/neo4j/dbms/DatabaseManagementSystemSettings.java @@ -21,35 +21,19 @@ import java.io.File; -import org.neo4j.configuration.Description; import org.neo4j.configuration.Internal; import org.neo4j.configuration.LoadableConfig; import org.neo4j.graphdb.config.Setting; +import org.neo4j.graphdb.factory.GraphDatabaseSettings; import static org.neo4j.kernel.configuration.Settings.PATH; -import static org.neo4j.kernel.configuration.Settings.STRING; import static org.neo4j.kernel.configuration.Settings.derivedSetting; -import static org.neo4j.kernel.configuration.Settings.pathSetting; -import static org.neo4j.kernel.configuration.Settings.setting; public class DatabaseManagementSystemSettings implements LoadableConfig { - @Description( "Name of the database to load" ) - public static final Setting active_database = setting( "dbms.active_database", STRING, "graph.db" ); - - @Description( "Path of the data directory. You must not configure more than one Neo4j installation to use the " + - "same data directory." ) - public static final Setting data_directory = pathSetting( "dbms.directories.data", "data" ); - - @Internal - public static final Setting database_path = derivedSetting( "unsupported.dbms.directories.database", - data_directory, active_database, - ( data, current ) -> new File( new File( data, "databases" ), current ), - PATH ); - @Internal public static final Setting auth_store_directory = derivedSetting( "unsupported.dbms.directories.auth", - data_directory, + GraphDatabaseSettings.data_directory, data -> new File( data, "dbms" ), PATH ); } diff --git a/community/dbms/src/main/java/org/neo4j/dbms/archive/Dumper.java b/community/dbms/src/main/java/org/neo4j/dbms/archive/Dumper.java index 69e471f3eb142..e7152d3b20adc 100644 --- a/community/dbms/src/main/java/org/neo4j/dbms/archive/Dumper.java +++ b/community/dbms/src/main/java/org/neo4j/dbms/archive/Dumper.java @@ -19,6 +19,11 @@ */ package org.neo4j.dbms.archive; +import org.apache.commons.compress.archivers.ArchiveEntry; +import org.apache.commons.compress.archivers.ArchiveOutputStream; +import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream; +import org.apache.commons.compress.compressors.gzip.GzipCompressorOutputStream; + import java.io.IOException; import java.io.InputStream; import java.nio.file.Files; @@ -26,11 +31,7 @@ import java.nio.file.StandardOpenOption; import java.util.function.Predicate; -import org.apache.commons.compress.archivers.ArchiveEntry; -import org.apache.commons.compress.archivers.ArchiveOutputStream; -import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream; -import org.apache.commons.compress.compressors.gzip.GzipCompressorOutputStream; - +import org.neo4j.commandline.Util; import org.neo4j.function.ThrowingAction; import static org.neo4j.dbms.archive.Utils.checkWritableDirectory; @@ -45,20 +46,31 @@ public class Dumper { - public void dump( Path root, Path archive, Predicate exclude ) throws IOException + public void dump( Path dbPath, Path transactionalLogsPath, Path archive, Predicate exclude ) + throws IOException { checkWritableDirectory( archive.getParent() ); try ( ArchiveOutputStream stream = openArchiveOut( archive ) ) { - Files.walkFileTree( root, - onlyMatching( not( exclude ), - throwExceptions( - onDirectory( dir -> dumpDirectory( root, stream, dir ), - onFile( file -> dumpFile( root, stream, file ), - justContinue() ) ) ) ) ); + visitPath( dbPath, exclude, stream ); + if ( !Util.isSameOrChildPath( dbPath, transactionalLogsPath ) ) + { + visitPath( transactionalLogsPath, exclude, stream ); + } } } + private void visitPath( Path transactionalLogsPath, Predicate exclude, ArchiveOutputStream stream ) + throws IOException + { + Files.walkFileTree( transactionalLogsPath, + onlyMatching( not( exclude ), + throwExceptions( + onDirectory( dir -> dumpDirectory( transactionalLogsPath, stream, dir ), + onFile( file -> dumpFile( transactionalLogsPath, stream, file ), + justContinue() ) ) ) ) ); + } + private static ArchiveOutputStream openArchiveOut( Path archive ) throws IOException { // StandardOpenOption.CREATE_NEW is important here because it atomically asserts that the file doesn't diff --git a/community/dbms/src/main/java/org/neo4j/dbms/archive/Loader.java b/community/dbms/src/main/java/org/neo4j/dbms/archive/Loader.java index a9113eb09dc7b..e6fa932c1af8c 100644 --- a/community/dbms/src/main/java/org/neo4j/dbms/archive/Loader.java +++ b/community/dbms/src/main/java/org/neo4j/dbms/archive/Loader.java @@ -19,41 +19,71 @@ */ package org.neo4j.dbms.archive; +import org.apache.commons.compress.archivers.ArchiveEntry; +import org.apache.commons.compress.archivers.ArchiveInputStream; +import org.apache.commons.compress.archivers.tar.TarArchiveInputStream; +import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream; + import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; import java.nio.file.FileAlreadyExistsException; +import java.nio.file.FileSystemException; import java.nio.file.Files; import java.nio.file.Path; +import java.nio.file.Paths; -import org.apache.commons.compress.archivers.ArchiveEntry; -import org.apache.commons.compress.archivers.ArchiveInputStream; -import org.apache.commons.compress.archivers.tar.TarArchiveInputStream; -import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream; +import org.neo4j.kernel.impl.transaction.log.files.TransactionLogFiles; import static java.nio.file.Files.exists; - import static org.neo4j.dbms.archive.Utils.checkWritableDirectory; public class Loader { - public void load( Path archive, Path destination ) throws IOException, IncorrectFormat + public void load( Path archive, Path databaseDestination, Path transactionLogsDirectory ) throws IOException, IncorrectFormat { - if ( exists( destination ) ) - { - throw new FileAlreadyExistsException( destination.toString() ); - } - checkWritableDirectory( destination.getParent() ); + validatePath( databaseDestination ); + validatePath( transactionLogsDirectory ); + + createDestination( databaseDestination ); + createDestination( transactionLogsDirectory ); + try ( ArchiveInputStream stream = openArchiveIn( archive ) ) { ArchiveEntry entry; while ( (entry = nextEntry( stream, archive )) != null ) { + Path destination = determineEntryDestination( entry, databaseDestination, transactionLogsDirectory ); loadEntry( destination, stream, entry ); } } } + private void createDestination( Path destination ) throws IOException + { + if ( !destination.toFile().exists() ) + { + Files.createDirectories( destination ); + } + } + + private void validatePath( Path path ) throws FileSystemException + { + if ( exists( path ) ) + { + throw new FileAlreadyExistsException( path.toString() ); + } + checkWritableDirectory( path.getParent() ); + } + + private static Path determineEntryDestination( ArchiveEntry entry, Path databaseDestination, + Path transactionLogsDirectory ) + { + String entryName = Paths.get( entry.getName() ).getFileName().toString(); + return TransactionLogFiles.DEFAULT_FILENAME_FILTER.accept( null, entryName ) ? transactionLogsDirectory + : databaseDestination; + } + private ArchiveEntry nextEntry( ArchiveInputStream stream, Path archive ) throws IncorrectFormat { try diff --git a/community/dbms/src/test/java/org/neo4j/commandline/dbms/CsvImporterTest.java b/community/dbms/src/test/java/org/neo4j/commandline/dbms/CsvImporterTest.java index e288fefe78194..d5a28df9cace2 100644 --- a/community/dbms/src/test/java/org/neo4j/commandline/dbms/CsvImporterTest.java +++ b/community/dbms/src/test/java/org/neo4j/commandline/dbms/CsvImporterTest.java @@ -30,7 +30,6 @@ import java.util.Map; import org.neo4j.commandline.admin.RealOutsideWorld; -import org.neo4j.dbms.DatabaseManagementSystemSettings; import org.neo4j.graphdb.factory.GraphDatabaseSettings; import org.neo4j.helpers.Args; import org.neo4j.kernel.configuration.Config; @@ -62,7 +61,7 @@ public void writesReportToSpecifiedReportFile() throws Exception { Config config = Config.builder() .withSettings( additionalConfig() ) - .withSetting( DatabaseManagementSystemSettings.database_path, dbDir.getAbsolutePath() ) + .withSetting( GraphDatabaseSettings.database_path, dbDir.getAbsolutePath() ) .withSetting( GraphDatabaseSettings.logs_directory, logDir.getAbsolutePath() ).build(); CsvImporter csvImporter = new CsvImporter( @@ -80,7 +79,7 @@ public void writesReportToSpecifiedReportFile() throws Exception private Map additionalConfig() { - return stringMap( DatabaseManagementSystemSettings.database_path.name(), getDatabasePath(), + return stringMap( GraphDatabaseSettings.database_path.name(), getDatabasePath(), GraphDatabaseSettings.logs_directory.name(), getLogsDirectory() ); } diff --git a/community/dbms/src/test/java/org/neo4j/commandline/dbms/DatabaseImporterTest.java b/community/dbms/src/test/java/org/neo4j/commandline/dbms/DatabaseImporterTest.java index 4eec3d376b8f0..432e8d763b1cc 100644 --- a/community/dbms/src/test/java/org/neo4j/commandline/dbms/DatabaseImporterTest.java +++ b/community/dbms/src/test/java/org/neo4j/commandline/dbms/DatabaseImporterTest.java @@ -30,7 +30,6 @@ import org.neo4j.commandline.admin.IncorrectUsage; import org.neo4j.commandline.admin.NullOutsideWorld; -import org.neo4j.dbms.DatabaseManagementSystemSettings; import org.neo4j.graphdb.GraphDatabaseService; import org.neo4j.graphdb.Transaction; import org.neo4j.graphdb.factory.GraphDatabaseSettings; @@ -128,7 +127,7 @@ private Config getConfigWith( File homeDir, String databaseName ) { HashMap additionalConfig = new HashMap<>(); additionalConfig.put( GraphDatabaseSettings.neo4j_home.name(), homeDir.toString() ); - additionalConfig.put( DatabaseManagementSystemSettings.active_database.name(), databaseName ); + additionalConfig.put( GraphDatabaseSettings.active_database.name(), databaseName ); return Config.defaults( additionalConfig ); } diff --git a/community/dbms/src/test/java/org/neo4j/commandline/dbms/DumpCommandTest.java b/community/dbms/src/test/java/org/neo4j/commandline/dbms/DumpCommandTest.java index d9c89cf8a3464..c58b7418025cb 100644 --- a/community/dbms/src/test/java/org/neo4j/commandline/dbms/DumpCommandTest.java +++ b/community/dbms/src/test/java/org/neo4j/commandline/dbms/DumpCommandTest.java @@ -43,6 +43,7 @@ import org.neo4j.commandline.admin.IncorrectUsage; import org.neo4j.commandline.admin.Usage; import org.neo4j.dbms.archive.Dumper; +import org.neo4j.graphdb.config.Setting; import org.neo4j.io.fs.DefaultFileSystemAbstraction; import org.neo4j.io.fs.FileSystemAbstraction; import org.neo4j.kernel.configuration.Config; @@ -66,8 +67,9 @@ import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; -import static org.neo4j.dbms.DatabaseManagementSystemSettings.data_directory; import static org.neo4j.dbms.archive.TestUtils.withPermissions; +import static org.neo4j.graphdb.factory.GraphDatabaseSettings.data_directory; +import static org.neo4j.graphdb.factory.GraphDatabaseSettings.logical_logs_location; public class DumpCommandTest { @@ -97,7 +99,8 @@ public void setUp() throws Exception public void shouldDumpTheDatabaseToTheArchive() throws Exception { execute( "foo.db" ); - verify( dumper ).dump( eq( homeDir.resolve( "data/databases/foo.db" ) ), eq( archive ), any() ); + verify( dumper ).dump( eq( homeDir.resolve( "data/databases/foo.db" ) ), + eq( homeDir.resolve( "data/databases/foo.db" ) ), eq( archive ), any() ); } @Test @@ -107,10 +110,25 @@ public void shouldCalculateTheDatabaseDirectoryFromConfig() throws Exception Path databaseDir = dataDir.resolve( "databases/foo.db" ); putStoreInDirectory( databaseDir ); Files.write( configDir.resolve( Config.DEFAULT_CONFIG_FILE_NAME ), - asList( format( "%s=%s", data_directory.name(), dataDir.toString().replace( '\\', '/' ) ) ) ); + asList( formatProperty( data_directory, dataDir ) ) ); execute( "foo.db" ); - verify( dumper ).dump( eq( databaseDir ), any(), any() ); + verify( dumper ).dump( eq( databaseDir ), eq( databaseDir ), any(), any() ); + } + + @Test + public void shouldCalculateTheTxLogDirectoryFromConfig() throws Exception + { + Path dataDir = testDirectory.directory( "some-other-path" ).toPath(); + Path txLogsDir = testDirectory.directory( "txLogsPath" ).toPath(); + Path databaseDir = dataDir.resolve( "databases/foo.db" ); + putStoreInDirectory( databaseDir ); + Files.write( configDir.resolve( Config.DEFAULT_CONFIG_FILE_NAME ), + asList( formatProperty( data_directory, dataDir ), + formatProperty( logical_logs_location, txLogsDir ) ) ); + + execute( "foo.db" ); + verify( dumper ).dump( eq( databaseDir ), eq( txLogsDir ), any(), any() ); } @Test @@ -132,7 +150,7 @@ public void shouldHandleDatabaseSymlink() throws Exception asList( format( "%s=%s", data_directory.name(), dataDir.toString().replace( '\\', '/' ) ) ) ); execute( "foo.db" ); - verify( dumper ).dump( eq( realDatabaseDir ), any(), any() ); + verify( dumper ).dump( eq( realDatabaseDir ), eq( realDatabaseDir ), any(), any() ); } @Test @@ -141,7 +159,7 @@ public void shouldCalculateTheArchiveNameIfPassedAnExistingDirectory() { File to = testDirectory.directory( "some-dir" ); new DumpCommand( homeDir, configDir, dumper ).execute( new String[]{"--database=" + "foo.db", "--to=" + to} ); - verify( dumper ).dump( any( Path.class ), eq( to.toPath().resolve( "foo.db.dump" ) ), any() ); + verify( dumper ).dump( any( Path.class ), any( Path.class ), eq( to.toPath().resolve( "foo.db.dump" ) ), any() ); } @Test @@ -149,7 +167,8 @@ public void shouldConvertToCanonicalPath() throws Exception { new DumpCommand( homeDir, configDir, dumper ) .execute( new String[]{"--database=" + "foo.db", "--to=foo.dump"} ); - verify( dumper ).dump( any( Path.class ), eq( Paths.get( new File( "foo.dump" ).getCanonicalPath() ) ), any() ); + verify( dumper ).dump( any( Path.class ), any( Path.class ), + eq( Paths.get( new File( "foo.dump" ).getCanonicalPath() ) ), any() ); } @Test @@ -158,7 +177,7 @@ public void shouldNotCalculateTheArchiveNameIfPassedAnExistingFile() { Files.createFile( archive ); execute( "foo.db" ); - verify( dumper ).dump( any(), eq( archive ), any() ); + verify( dumper ).dump( any(), any(), eq( archive ), any() ); } @Test @@ -190,7 +209,7 @@ public void shouldReleaseTheStoreLockAfterDumping() throws Exception @Test public void shouldReleaseTheStoreLockEvenIfThereIsAnError() throws Exception { - doThrow( IOException.class ).when( dumper ).dump( any(), any(), any() ); + doThrow( IOException.class ).when( dumper ).dump( any(), any(), any(), any() ); try { @@ -213,7 +232,7 @@ public void shouldNotAccidentallyCreateTheDatabaseDirectoryAsASideEffectOfStoreL { assertThat( Files.exists( databaseDirectory ), equalTo( false ) ); return null; - } ).when( dumper ).dump( any(), any(), any() ); + } ).when( dumper ).dump( any(), any(), any(), any() ); execute( "foo.db" ); } @@ -249,11 +268,11 @@ public void shouldExcludeTheStoreLockFromTheArchiveToAvoidProblemsWithReadingLoc doAnswer( invocation -> { //noinspection unchecked - Predicate exclude = invocation.getArgument( 2 ); + Predicate exclude = invocation.getArgument( 3 ); assertThat( exclude.test( Paths.get( StoreLocker.STORE_LOCK_FILENAME ) ), is( true ) ); assertThat( exclude.test( Paths.get( "some-other-file" ) ), is( false ) ); return null; - } ).when( dumper ).dump( any(), any(), any() ); + } ).when( dumper ).dump(any(), any(), any(), any() ); execute( "foo.db" ); } @@ -265,10 +284,10 @@ public void shouldDefaultToGraphDB() throws Exception Path databaseDir = dataDir.resolve( "databases/graph.db" ); putStoreInDirectory( databaseDir ); Files.write( configDir.resolve( Config.DEFAULT_CONFIG_FILE_NAME ), - asList( format( "%s=%s", data_directory.name(), dataDir.toString().replace( '\\', '/' ) ) ) ); + asList( formatProperty( data_directory, dataDir ) ) ); new DumpCommand( homeDir, configDir, dumper ).execute( new String[]{"--to=" + archive} ); - verify( dumper ).dump( eq( databaseDir ), any(), any() ); + verify( dumper ).dump( eq( databaseDir ), eq( databaseDir ), any(), any() ); } @Test @@ -288,7 +307,7 @@ public void shouldObjectIfTheArchiveArgumentIsMissing() throws Exception @Test public void shouldGiveAClearErrorIfTheArchiveAlreadyExists() throws Exception { - doThrow( new FileAlreadyExistsException( "the-archive-path" ) ).when( dumper ).dump( any(), any(), any() ); + doThrow( new FileAlreadyExistsException( "the-archive-path" ) ).when( dumper ).dump( any(), any(), any(), any() ); try { execute( "foo.db" ); @@ -317,7 +336,7 @@ public void shouldGiveAClearMessageIfTheDatabaseDoesntExist() throws Exception @Test public void shouldGiveAClearMessageIfTheArchivesParentDoesntExist() throws Exception { - doThrow( new NoSuchFileException( archive.getParent().toString() ) ).when( dumper ).dump( any(), any(), any() ); + doThrow( new NoSuchFileException( archive.getParent().toString() ) ).when( dumper ).dump(any(), any(), any(), any() ); try { execute( "foo.db" ); @@ -331,10 +350,10 @@ public void shouldGiveAClearMessageIfTheArchivesParentDoesntExist() throws Excep } @Test - public void shouldWrapIOExceptionsCarefulllyBecauseCriticalInformationIsOftenEncodedInTheirNameButMissingFromTheirMessage() + public void shouldWrapIOExceptionsCarefullyBecauseCriticalInformationIsOftenEncodedInTheirNameButMissingFromTheirMessage() throws Exception { - doThrow( new IOException( "the-message" ) ).when( dumper ).dump( any(), any(), any() ); + doThrow( new IOException( "the-message" ) ).when( dumper ).dump(any(), any(), any(), any() ); try { execute( "foo.db" ); @@ -383,7 +402,7 @@ private void execute( final String database ) throws IncorrectUsage, CommandFail .execute( new String[]{"--database=" + database, "--to=" + archive} ); } - private void assertCanLockStore( Path databaseDirectory ) throws IOException + private static void assertCanLockStore( Path databaseDirectory ) throws IOException { try ( FileSystemAbstraction fileSystem = new DefaultFileSystemAbstraction(); StoreLocker storeLocker = new StoreLocker( fileSystem, databaseDirectory.toFile() ) ) @@ -392,10 +411,15 @@ private void assertCanLockStore( Path databaseDirectory ) throws IOException } } - private void putStoreInDirectory( Path storeDir ) throws IOException + private static void putStoreInDirectory( Path storeDir ) throws IOException { Files.createDirectories( storeDir ); Path storeFile = storeDir.resolve( StoreFileType.STORE.augment( MetaDataStore.DEFAULT_NAME ) ); Files.createFile( storeFile ); } + + private static String formatProperty( Setting setting, Path path ) + { + return format( "%s=%s", setting.name(), path.toString().replace( '\\', '/' ) ); + } } diff --git a/community/dbms/src/test/java/org/neo4j/commandline/dbms/LoadCommandTest.java b/community/dbms/src/test/java/org/neo4j/commandline/dbms/LoadCommandTest.java index 023ee3f4aa7c9..53dd0417d6674 100644 --- a/community/dbms/src/test/java/org/neo4j/commandline/dbms/LoadCommandTest.java +++ b/community/dbms/src/test/java/org/neo4j/commandline/dbms/LoadCommandTest.java @@ -42,6 +42,7 @@ import org.neo4j.commandline.admin.Usage; import org.neo4j.dbms.archive.IncorrectFormat; import org.neo4j.dbms.archive.Loader; +import org.neo4j.graphdb.config.Setting; import org.neo4j.helpers.ArrayUtil; import org.neo4j.io.fs.DefaultFileSystemAbstraction; import org.neo4j.io.fs.FileSystemAbstraction; @@ -63,7 +64,8 @@ import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; -import static org.neo4j.dbms.DatabaseManagementSystemSettings.data_directory; +import static org.neo4j.graphdb.factory.GraphDatabaseSettings.data_directory; +import static org.neo4j.graphdb.factory.GraphDatabaseSettings.logical_logs_location; public class LoadCommandTest { @@ -87,7 +89,7 @@ public void setUp() throws Exception public void shouldLoadTheDatabaseFromTheArchive() throws CommandFailed, IncorrectUsage, IOException, IncorrectFormat { execute( "foo.db" ); - verify( loader ).load( archive, homeDir.resolve( "data/databases/foo.db" ) ); + verify( loader ).load( archive, homeDir.resolve( "data/databases/foo.db" ), homeDir.resolve( "data/databases/foo.db" ) ); } @Test @@ -98,10 +100,24 @@ public void shouldCalculateTheDatabaseDirectoryFromConfig() Path databaseDir = dataDir.resolve( "databases/foo.db" ); Files.createDirectories( databaseDir ); Files.write( configDir.resolve( Config.DEFAULT_CONFIG_FILE_NAME ), - asList( format( "%s=%s", data_directory.name(), dataDir.toString().replace( '\\', '/' ) ) ) ); + asList( formatProperty( data_directory, dataDir ) ) ); execute( "foo.db" ); - verify( loader ).load( any(), eq( databaseDir ) ); + verify( loader ).load( any(), eq( databaseDir ), eq( databaseDir ) ); + } + + @Test + public void shouldCalculateTheTxLogDirectoryFromConfig() throws Exception + { + Path dataDir = testDirectory.directory( "some-other-path" ).toPath(); + Path txLogsDir = testDirectory.directory( "txLogsPath" ).toPath(); + Path databaseDir = dataDir.resolve( "databases/foo.db" ); + Files.write( configDir.resolve( Config.DEFAULT_CONFIG_FILE_NAME ), + asList( formatProperty( data_directory, dataDir ), + formatProperty( logical_logs_location, txLogsDir ) ) ); + + execute( "foo.db" ); + verify( loader ).load( any(), eq( databaseDir ), eq( txLogsDir ) ); } @Test @@ -121,10 +137,10 @@ public void shouldHandleSymlinkToDatabaseDir() throws IOException, CommandFailed Files.createSymbolicLink( databaseDir, realDatabaseDir ); Files.write( configDir.resolve( Config.DEFAULT_CONFIG_FILE_NAME ), - asList( format( "%s=%s", data_directory.name(), dataDir.toString().replace( '\\', '/' ) ) ) ); + asList( formatProperty( data_directory, dataDir ) ) ); execute( "foo.db" ); - verify( loader ).load( any(), eq( realDatabaseDir ) ); + verify( loader ).load( any(), eq( realDatabaseDir ), eq( realDatabaseDir ) ); } @Test @@ -134,12 +150,12 @@ public void shouldMakeFromCanonical() throws IOException, CommandFailed, Incorre Path databaseDir = dataDir.resolve( "databases/foo.db" ); Files.createDirectories( databaseDir ); Files.write( configDir.resolve( Config.DEFAULT_CONFIG_FILE_NAME ), - asList( format( "%s=%s", data_directory.name(), dataDir.toString().replace( '\\', '/' ) ) ) ); + asList( formatProperty( data_directory, dataDir ) ) ); new LoadCommand( homeDir, configDir, loader ) .execute( ArrayUtil.concat( new String[]{"--database=foo.db", "--from=foo.dump"} ) ); - verify( loader ).load( eq( Paths.get( new File( "foo.dump" ).getCanonicalPath() ) ), any() ); + verify( loader ).load( eq( Paths.get( new File( "foo.dump" ).getCanonicalPath() ) ), any(), any() ); } @Test @@ -153,7 +169,7 @@ public void shouldDeleteTheOldDatabaseIfForceArgumentIsProvided() { assertThat( Files.exists( databaseDirectory ), equalTo( false ) ); return null; - } ).when( loader ).load( any(), any() ); + } ).when( loader ).load( any(), any(), any() ); execute( "foo.db", "--force" ); } @@ -169,7 +185,7 @@ public void shouldNotDeleteTheOldDatabaseIfForceArgumentIsNotProvided() { assertThat( Files.exists( databaseDirectory ), equalTo( true ) ); return null; - } ).when( loader ).load( any(), any() ); + } ).when( loader ).load( any(), any(), any() ); execute( "foo.db" ); } @@ -200,7 +216,7 @@ public void shouldDefaultToGraphDb() throws Exception Files.createDirectories( databaseDir ); new LoadCommand( homeDir, configDir, loader ).execute( new String[]{"--from=something"} ); - verify( loader ).load( any(), eq( databaseDir ) ); + verify( loader ).load( any(), eq( databaseDir ), eq( databaseDir ) ); } @Test @@ -220,7 +236,7 @@ public void shouldObjectIfTheArchiveArgumentIsMissing() throws Exception @Test public void shouldGiveAClearMessageIfTheArchiveDoesntExist() throws IOException, IncorrectFormat, IncorrectUsage { - doThrow( new NoSuchFileException( archive.toString() ) ).when( loader ).load( any(), any() ); + doThrow( new NoSuchFileException( archive.toString() ) ).when( loader ).load( any(), any(), any() ); try { execute( null ); @@ -235,7 +251,7 @@ public void shouldGiveAClearMessageIfTheArchiveDoesntExist() throws IOException, @Test public void shouldGiveAClearMessageIfTheDatabaseAlreadyExists() throws IOException, IncorrectFormat, IncorrectUsage { - doThrow( FileAlreadyExistsException.class ).when( loader ).load( any(), any() ); + doThrow( FileAlreadyExistsException.class ).when( loader ).load( any(), any(), any() ); try { execute( "foo.db" ); @@ -251,7 +267,7 @@ public void shouldGiveAClearMessageIfTheDatabaseAlreadyExists() throws IOExcepti public void shouldGiveAClearMessageIfTheDatabasesDirectoryIsNotWritable() throws IOException, IncorrectFormat, IncorrectUsage { - doThrow( AccessDeniedException.class ).when( loader ).load( any(), any() ); + doThrow( AccessDeniedException.class ).when( loader ).load( any(), any(), any() ); try { execute( null ); @@ -265,10 +281,10 @@ public void shouldGiveAClearMessageIfTheDatabasesDirectoryIsNotWritable() } @Test - public void shouldWrapIOExceptionsCarefulllyBecauseCriticalInformationIsOftenEncodedInTheirNameButMissingFromTheirMessage() + public void shouldWrapIOExceptionsCarefullyBecauseCriticalInformationIsOftenEncodedInTheirNameButMissingFromTheirMessage() throws IOException, IncorrectUsage, IncorrectFormat { - doThrow( new FileSystemException( "the-message" ) ).when( loader ).load( any(), any() ); + doThrow( new FileSystemException( "the-message" ) ).when( loader ).load( any(), any(), any() ); try { execute( null ); @@ -283,7 +299,7 @@ public void shouldWrapIOExceptionsCarefulllyBecauseCriticalInformationIsOftenEnc @Test public void shouldThrowIfTheArchiveFormatIsInvalid() throws IOException, IncorrectUsage, IncorrectFormat { - doThrow( IncorrectFormat.class ).when( loader ).load( any(), any() ); + doThrow( IncorrectFormat.class ).when( loader ).load( any(), any(), any() ); try { execute( null ); @@ -335,4 +351,9 @@ private void execute( String database, String... otherArgs ) throws IncorrectUsa new LoadCommand( homeDir, configDir, loader ) .execute( ArrayUtil.concat( new String[]{"--database=" + database, "--from=" + archive}, otherArgs ) ); } + + private static String formatProperty( Setting setting, Path path ) + { + return format( "%s=%s", setting.name(), path.toString().replace( '\\', '/' ) ); + } } diff --git a/community/dbms/src/test/java/org/neo4j/commandline/dbms/UtilTest.java b/community/dbms/src/test/java/org/neo4j/commandline/dbms/UtilTest.java index d38d9e9a5379b..06b6c5e730187 100644 --- a/community/dbms/src/test/java/org/neo4j/commandline/dbms/UtilTest.java +++ b/community/dbms/src/test/java/org/neo4j/commandline/dbms/UtilTest.java @@ -19,15 +19,24 @@ */ package org.neo4j.commandline.dbms; +import org.junit.Rule; import org.junit.Test; import org.neo4j.commandline.Util; +import org.neo4j.test.rule.TestDirectory; +import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; +import static org.neo4j.commandline.Util.isSameOrChildFile; +import static org.neo4j.commandline.Util.isSameOrChildPath; import static org.neo4j.commandline.Util.neo4jVersion; public class UtilTest { + @Rule + public final TestDirectory directory = TestDirectory.testDirectory(); + @Test public void canonicalPath() throws Exception { @@ -39,4 +48,29 @@ public void returnsAVersion() throws Exception { assertNotNull( "A version should be returned", neo4jVersion() ); } + + @Test + public void correctlyIdentifySameOrChildFile() + { + assertTrue( isSameOrChildFile( directory.directory(), directory.directory( "a" ) ) ); + assertTrue( isSameOrChildFile( directory.directory(), directory.directory() ) ); + assertTrue( isSameOrChildFile( directory.directory( "/a/./b" ), directory.directory( "a/b" ) ) ); + assertTrue( isSameOrChildFile( directory.directory( "a/b" ), directory.directory( "/a/./b" ) ) ); + + assertFalse( isSameOrChildFile( directory.directory( "a" ), directory.directory( "b" ) ) ); + } + + @Test + public void correctlyIdentifySameOrChildPath() + { + assertTrue( isSameOrChildPath( directory.directory().toPath(), directory.directory( "a" ).toPath() ) ); + assertTrue( isSameOrChildPath( directory.directory().toPath(), directory.directory().toPath() ) ); + assertTrue( isSameOrChildPath( directory.directory( "/a/./b" ).toPath(), + directory.directory( "a/b" ).toPath() ) ); + assertTrue( isSameOrChildPath( directory.directory( "a/b" ).toPath(), + directory.directory( "/a/./b" ).toPath() ) ); + + assertFalse( isSameOrChildPath( directory.directory( "a" ).toPath(), + directory.directory( "b" ).toPath() ) ); + } } diff --git a/community/dbms/src/test/java/org/neo4j/dbms/DatabaseManagementSystemSettingsTest.java b/community/dbms/src/test/java/org/neo4j/dbms/DatabaseManagementSystemSettingsTest.java index d5b604db222de..56ef58480ca0b 100644 --- a/community/dbms/src/test/java/org/neo4j/dbms/DatabaseManagementSystemSettingsTest.java +++ b/community/dbms/src/test/java/org/neo4j/dbms/DatabaseManagementSystemSettingsTest.java @@ -23,6 +23,7 @@ import java.io.File; +import org.neo4j.graphdb.factory.GraphDatabaseSettings; import org.neo4j.kernel.configuration.Config; import static org.hamcrest.core.IsEqual.equalTo; @@ -33,8 +34,8 @@ public class DatabaseManagementSystemSettingsTest @Test public void shouldPutDatabaseDirectoriesIntoDataDatabases() { - Config config = Config.defaults( DatabaseManagementSystemSettings.data_directory, "the-data-directory" ); - assertThat( config.get( DatabaseManagementSystemSettings.database_path ), + Config config = Config.defaults( GraphDatabaseSettings.data_directory, "the-data-directory" ); + assertThat( config.get( GraphDatabaseSettings.database_path ), equalTo( new File( "the-data-directory/databases/graph.db" ) ) ); } } diff --git a/community/dbms/src/test/java/org/neo4j/dbms/archive/ArchiveTest.java b/community/dbms/src/test/java/org/neo4j/dbms/archive/ArchiveTest.java index bc4d113c08075..2221673f6dcaa 100644 --- a/community/dbms/src/test/java/org/neo4j/dbms/archive/ArchiveTest.java +++ b/community/dbms/src/test/java/org/neo4j/dbms/archive/ArchiveTest.java @@ -30,6 +30,7 @@ import java.util.Map; import org.neo4j.function.Predicates; +import org.neo4j.kernel.impl.transaction.log.files.TransactionLogFiles; import org.neo4j.test.rule.TestDirectory; import static java.nio.file.Files.isDirectory; @@ -121,9 +122,9 @@ public void shouldExcludeFilesMatchedByTheExclusionPredicate() throws IOExceptio Files.write( directory.resolve( "another-file" ), new byte[0] ); Path archive = testDirectory.file( "the-archive.dump" ).toPath(); - new Dumper().dump( directory, archive, path -> path.getFileName().toString().equals( "another-file" ) ); + new Dumper().dump( directory, directory, archive, path -> path.getFileName().toString().equals( "another-file" ) ); Path newDirectory = testDirectory.file( "the-new-directory" ).toPath(); - new Loader().load( archive, newDirectory ); + new Loader().load( archive, newDirectory, newDirectory ); Path expectedOutput = testDirectory.directory( "expected-output" ).toPath(); Files.createDirectories( expectedOutput ); @@ -141,9 +142,9 @@ public void shouldExcludeWholeDirectoriesMatchedByTheExclusionPredicate() throws Files.write( subdir.resolve( "a-file" ), new byte[0] ); Path archive = testDirectory.file( "the-archive.dump" ).toPath(); - new Dumper().dump( directory, archive, path -> path.getFileName().toString().equals( "subdir" ) ); + new Dumper().dump( directory, directory, archive, path -> path.getFileName().toString().equals( "subdir" ) ); Path newDirectory = testDirectory.file( "the-new-directory" ).toPath(); - new Loader().load( archive, newDirectory ); + new Loader().load( archive, newDirectory, newDirectory ); Path expectedOutput = testDirectory.directory( "expected-output" ).toPath(); Files.createDirectories( expectedOutput ); @@ -151,12 +152,38 @@ public void shouldExcludeWholeDirectoriesMatchedByTheExclusionPredicate() throws assertEquals( describeRecursively( expectedOutput ), describeRecursively( newDirectory ) ); } + @Test + public void dumpAndLoadTransactionLogsFromCustomLocations() throws IOException, IncorrectFormat + { + Path directory = testDirectory.directory( "dbDirectory" ).toPath(); + Path txLogsDirectory = testDirectory.directory( "txLogsDirectory" ).toPath(); + Files.write( directory.resolve( "dbfile" ), new byte[0] ); + Files.write( txLogsDirectory.resolve( TransactionLogFiles.DEFAULT_NAME + ".0" ), new byte[0] ); + + Path archive = testDirectory.file( "the-archive.dump" ).toPath(); + new Dumper().dump( directory, txLogsDirectory, archive, Predicates.alwaysFalse() ); + Path newDirectory = testDirectory.file( "the-new-directory" ).toPath(); + Path newTxLogsDirectory = testDirectory.file( "newTxLogsDirectory" ).toPath(); + new Loader().load( archive, newDirectory, newTxLogsDirectory ); + + Path expectedOutput = testDirectory.directory( "expected-output" ).toPath(); + Files.createDirectories( expectedOutput ); + Files.write( expectedOutput.resolve( "dbfile" ), new byte[0] ); + + Path expectedTxLogs = testDirectory.directory( "expectedTxLogs" ).toPath(); + Files.createDirectories( expectedTxLogs ); + Files.write( expectedTxLogs.resolve( TransactionLogFiles.DEFAULT_NAME + ".0" ), new byte[0] ); + + assertEquals( describeRecursively( expectedOutput ), describeRecursively( newDirectory ) ); + assertEquals( describeRecursively( expectedTxLogs ), describeRecursively( newTxLogsDirectory ) ); + } + private void assertRoundTrips( Path oldDirectory ) throws IOException, IncorrectFormat { Path archive = testDirectory.file( "the-archive.dump" ).toPath(); - new Dumper().dump( oldDirectory, archive, Predicates.alwaysFalse() ); + new Dumper().dump( oldDirectory, oldDirectory, archive, Predicates.alwaysFalse() ); Path newDirectory = testDirectory.file( "the-new-directory" ).toPath(); - new Loader().load( archive, newDirectory ); + new Loader().load( archive, newDirectory, newDirectory ); assertEquals( describeRecursively( oldDirectory ), describeRecursively( newDirectory ) ); } diff --git a/community/dbms/src/test/java/org/neo4j/dbms/archive/DumperTest.java b/community/dbms/src/test/java/org/neo4j/dbms/archive/DumperTest.java index 734e8a90a2406..834654638f945 100644 --- a/community/dbms/src/test/java/org/neo4j/dbms/archive/DumperTest.java +++ b/community/dbms/src/test/java/org/neo4j/dbms/archive/DumperTest.java @@ -19,6 +19,10 @@ */ package org.neo4j.dbms.archive; +import org.apache.commons.lang3.SystemUtils; +import org.junit.Rule; +import org.junit.Test; + import java.io.Closeable; import java.io.IOException; import java.nio.file.AccessDeniedException; @@ -28,15 +32,10 @@ import java.nio.file.NoSuchFileException; import java.nio.file.Path; -import org.apache.commons.lang3.SystemUtils; -import org.junit.Rule; -import org.junit.Test; - import org.neo4j.function.Predicates; import org.neo4j.test.rule.TestDirectory; import static java.util.Collections.emptySet; - import static org.junit.Assert.assertEquals; import static org.junit.Assert.fail; import static org.junit.Assume.assumeFalse; @@ -54,7 +53,7 @@ public void shouldGiveAClearErrorIfTheArchiveAlreadyExists() throws IOException Files.write( archive, new byte[0] ); try { - new Dumper().dump( directory, archive, Predicates.alwaysFalse() ); + new Dumper().dump( directory, directory, archive, Predicates.alwaysFalse() ); fail( "Expected an exception" ); } catch ( FileAlreadyExistsException e ) @@ -70,7 +69,7 @@ public void shouldGiveAClearErrorMessageIfTheDirectoryDoesntExist() throws IOExc Path archive = testDirectory.file( "the-archive.dump" ).toPath(); try { - new Dumper().dump( directory, archive, Predicates.alwaysFalse() ); + new Dumper().dump( directory, directory, archive, Predicates.alwaysFalse() ); fail( "Expected an exception" ); } catch ( NoSuchFileException e ) @@ -86,7 +85,7 @@ public void shouldGiveAClearErrorMessageIfTheArchivesParentDirectoryDoesntExist( Path archive = testDirectory.file( "subdir/the-archive.dump" ).toPath(); try { - new Dumper().dump( directory, archive, Predicates.alwaysFalse() ); + new Dumper().dump( directory, directory, archive, Predicates.alwaysFalse() ); fail( "Expected an exception" ); } catch ( NoSuchFileException e ) @@ -103,7 +102,7 @@ public void shouldGiveAClearErrorMessageIfTheArchivesParentDirectoryIsAFile() th Files.write( archive.getParent(), new byte[0] ); try { - new Dumper().dump( directory, archive, Predicates.alwaysFalse() ); + new Dumper().dump( directory, directory, archive, Predicates.alwaysFalse() ); fail( "Expected an exception" ); } catch ( FileSystemException e ) @@ -122,7 +121,7 @@ public void shouldGiveAClearErrorMessageIfTheArchivesParentDirectoryIsNotWritabl Files.createDirectories( archive.getParent() ); try ( Closeable ignored = TestUtils.withPermissions( archive.getParent(), emptySet() ) ) { - new Dumper().dump( directory, archive, Predicates.alwaysFalse() ); + new Dumper().dump( directory, directory, archive, Predicates.alwaysFalse() ); fail( "Expected an exception" ); } catch ( AccessDeniedException e ) diff --git a/community/dbms/src/test/java/org/neo4j/dbms/archive/LoaderTest.java b/community/dbms/src/test/java/org/neo4j/dbms/archive/LoaderTest.java index 7afd0298dcc62..0521d28b40313 100644 --- a/community/dbms/src/test/java/org/neo4j/dbms/archive/LoaderTest.java +++ b/community/dbms/src/test/java/org/neo4j/dbms/archive/LoaderTest.java @@ -19,6 +19,11 @@ */ package org.neo4j.dbms.archive; +import org.apache.commons.compress.compressors.gzip.GzipCompressorOutputStream; +import org.apache.commons.lang3.SystemUtils; +import org.junit.Rule; +import org.junit.Test; + import java.io.Closeable; import java.io.IOException; import java.nio.file.AccessDeniedException; @@ -29,20 +34,13 @@ import java.nio.file.Path; import java.util.Random; -import org.apache.commons.compress.compressors.gzip.GzipCompressorOutputStream; -import org.apache.commons.lang3.SystemUtils; -import org.junit.Rule; -import org.junit.Test; - import org.neo4j.test.rule.TestDirectory; import static java.util.Arrays.asList; import static java.util.Collections.emptySet; - import static org.junit.Assert.assertEquals; import static org.junit.Assert.fail; import static org.junit.Assume.assumeFalse; - import static org.neo4j.dbms.archive.TestUtils.withPermissions; public class LoaderTest @@ -57,7 +55,7 @@ public void shouldGiveAClearErrorMessageIfTheArchiveDoesntExist() throws IOExcep Path destination = testDirectory.file( "the-destination" ).toPath(); try { - new Loader().load( archive, destination ); + new Loader().load( archive, destination, destination ); fail( "Expected an exception" ); } catch ( NoSuchFileException e ) @@ -74,7 +72,7 @@ public void shouldGiveAClearErrorMessageIfTheArchiveIsNotInGzipFormat() throws I Path destination = testDirectory.file( "the-destination" ).toPath(); try { - new Loader().load( archive, destination ); + new Loader().load( archive, destination, destination ); fail( "Expected an exception" ); } catch ( IncorrectFormat e ) @@ -98,7 +96,7 @@ public void shouldGiveAClearErrorMessageIfTheArchiveIsNotInTarFormat() throws IO Path destination = testDirectory.file( "the-destination" ).toPath(); try { - new Loader().load( archive, destination ); + new Loader().load( archive, destination, destination ); fail( "Expected an exception" ); } catch ( IncorrectFormat e ) @@ -114,7 +112,7 @@ public void shouldGiveAClearErrorIfTheDestinationAlreadyExists() throws IOExcept Path destination = testDirectory.directory( "the-destination" ).toPath(); try { - new Loader().load( archive, destination ); + new Loader().load( archive, destination, destination ); fail( "Expected an exception" ); } catch ( FileAlreadyExistsException e ) @@ -123,6 +121,23 @@ public void shouldGiveAClearErrorIfTheDestinationAlreadyExists() throws IOExcept } } + @Test + public void shouldGiveAClearErrorIfTheDestinationTxLogAlreadyExists() throws IOException, IncorrectFormat + { + Path archive = testDirectory.file( "the-archive.dump" ).toPath(); + Path destination = testDirectory.file( "the-destination" ).toPath(); + Path txLogsDestination = testDirectory.directory( "txLogsDestination" ).toPath(); + try + { + new Loader().load( archive, destination, txLogsDestination ); + fail( "Expected an exception" ); + } + catch ( FileAlreadyExistsException e ) + { + assertEquals( txLogsDestination.toString(), e.getMessage() ); + } + } + @Test public void shouldGiveAClearErrorMessageIfTheDestinationsParentDirectoryDoesntExist() throws IOException, IncorrectFormat @@ -131,7 +146,7 @@ public void shouldGiveAClearErrorMessageIfTheDestinationsParentDirectoryDoesntEx Path destination = testDirectory.directory( "subdir/the-destination" ).toPath(); try { - new Loader().load( archive, destination ); + new Loader().load( archive, destination, destination ); fail( "Expected an exception" ); } catch ( NoSuchFileException e ) @@ -140,6 +155,24 @@ public void shouldGiveAClearErrorMessageIfTheDestinationsParentDirectoryDoesntEx } } + @Test + public void shouldGiveAClearErrorMessageIfTheTxLogsParentDirectoryDoesntExist() + throws IOException, IncorrectFormat + { + Path archive = testDirectory.file( "the-archive.dump" ).toPath(); + Path destination = testDirectory.file( "destination" ).toPath(); + Path txLogsDestination = testDirectory.directory( "subdir/txLogs" ).toPath(); + try + { + new Loader().load( archive, destination, txLogsDestination ); + fail( "Expected an exception" ); + } + catch ( NoSuchFileException e ) + { + assertEquals( txLogsDestination.getParent().toString(), e.getMessage() ); + } + } + @Test public void shouldGiveAClearErrorMessageIfTheDestinationsParentDirectoryIsAFile() throws IOException, IncorrectFormat @@ -149,7 +182,7 @@ public void shouldGiveAClearErrorMessageIfTheDestinationsParentDirectoryIsAFile( Files.write( destination.getParent(), new byte[0] ); try { - new Loader().load( archive, destination ); + new Loader().load( archive, destination, destination ); fail( "Expected an exception" ); } catch ( FileSystemException e ) @@ -169,7 +202,7 @@ public void shouldGiveAClearErrorMessageIfTheDestinationsParentDirectoryIsNotWri Files.createDirectories( destination.getParent() ); try ( Closeable ignored = withPermissions( destination.getParent(), emptySet() ) ) { - new Loader().load( archive, destination ); + new Loader().load( archive, destination, destination ); fail( "Expected an exception" ); } catch ( AccessDeniedException e ) @@ -177,4 +210,25 @@ public void shouldGiveAClearErrorMessageIfTheDestinationsParentDirectoryIsNotWri assertEquals( destination.getParent().toString(), e.getMessage() ); } } + + @Test + public void shouldGiveAClearErrorMessageIfTheTxLogsParentDirectoryIsNotWritable() + throws IOException, IncorrectFormat + { + assumeFalse( "We haven't found a way to reliably tests permissions on Windows", SystemUtils.IS_OS_WINDOWS ); + + Path archive = testDirectory.file( "the-archive.dump" ).toPath(); + Path destination = testDirectory.file( "destination" ).toPath(); + Path txLogsDrectory = testDirectory.directory( "subdir/txLogs" ).toPath(); + Files.createDirectories( txLogsDrectory.getParent() ); + try ( Closeable ignored = withPermissions( txLogsDrectory.getParent(), emptySet() ) ) + { + new Loader().load( archive, destination, txLogsDrectory ); + fail( "Expected an exception" ); + } + catch ( AccessDeniedException e ) + { + assertEquals( txLogsDrectory.getParent().toString(), e.getMessage() ); + } + } } diff --git a/community/io/src/main/java/org/neo4j/io/fs/DefaultFileSystemAbstraction.java b/community/io/src/main/java/org/neo4j/io/fs/DefaultFileSystemAbstraction.java index 36e866ede85d3..fcd156d27a6e1 100644 --- a/community/io/src/main/java/org/neo4j/io/fs/DefaultFileSystemAbstraction.java +++ b/community/io/src/main/java/org/neo4j/io/fs/DefaultFileSystemAbstraction.java @@ -178,6 +178,12 @@ public void moveToDirectory( File file, File toDirectory ) throws IOException FileUtils.moveFileToDirectory( file, toDirectory ); } + @Override + public void copyToDirectory( File file, File toDirectory ) throws IOException + { + FileUtils.copyFileToDirectory( file, toDirectory ); + } + @Override public void copyFile( File from, File to ) throws IOException { diff --git a/community/io/src/main/java/org/neo4j/io/fs/DelegateFileSystemAbstraction.java b/community/io/src/main/java/org/neo4j/io/fs/DelegateFileSystemAbstraction.java index b7221027d719c..518673c63e235 100644 --- a/community/io/src/main/java/org/neo4j/io/fs/DelegateFileSystemAbstraction.java +++ b/community/io/src/main/java/org/neo4j/io/fs/DelegateFileSystemAbstraction.java @@ -227,6 +227,12 @@ public void moveToDirectory( File file, File toDirectory ) throws IOException Files.move( path( file ), path( toDirectory ).resolve( path( file.getName() ) ) ); } + @Override + public void copyToDirectory( File file, File toDirectory ) throws IOException + { + Files.copy( path( file ), toDirectory.toPath().resolve( file.getName() ) ); + } + @Override public void copyFile( File from, File to ) throws IOException { diff --git a/community/io/src/main/java/org/neo4j/io/fs/FileSystemAbstraction.java b/community/io/src/main/java/org/neo4j/io/fs/FileSystemAbstraction.java index 03966217f08ff..22715493418ba 100644 --- a/community/io/src/main/java/org/neo4j/io/fs/FileSystemAbstraction.java +++ b/community/io/src/main/java/org/neo4j/io/fs/FileSystemAbstraction.java @@ -82,6 +82,8 @@ public interface FileSystemAbstraction extends Closeable void moveToDirectory( File file, File toDirectory ) throws IOException; + void copyToDirectory( File file, File toDirectory ) throws IOException; + void copyFile( File from, File to ) throws IOException; void copyRecursively( File fromDirectory, File toDirectory ) throws IOException; diff --git a/community/io/src/main/java/org/neo4j/io/fs/FileUtils.java b/community/io/src/main/java/org/neo4j/io/fs/FileUtils.java index 2ec14011b416d..c4e0113f07728 100644 --- a/community/io/src/main/java/org/neo4j/io/fs/FileUtils.java +++ b/community/io/src/main/java/org/neo4j/io/fs/FileUtils.java @@ -51,6 +51,7 @@ import java.nio.file.Path; import java.nio.file.Paths; import java.nio.file.SimpleFileVisitor; +import java.nio.file.StandardCopyOption; import java.nio.file.StandardOpenOption; import java.nio.file.attribute.BasicFileAttributes; @@ -190,6 +191,30 @@ public static File moveFileToDirectory( File toMove, File targetDirectory ) thro return target; } + /** + * Utility method that copy a file from its current location to the + * provided target directory. + * + * @param file file that needs to be copied. + * @param targetDirectory the destination directory + * @throws IOException + */ + public static void copyFileToDirectory( File file, File targetDirectory ) throws IOException + { + if ( !targetDirectory.exists() ) + { + Files.createDirectories( targetDirectory.toPath() ); + } + if ( !targetDirectory.isDirectory() ) + { + throw new IllegalArgumentException( + "Move target must be a directory, not " + targetDirectory ); + } + + File target = new File( targetDirectory, file.getName() ); + copyFile( file, target ); + } + public static void renameFile( File srcFile, File renameToFile, CopyOption... copyOptions ) throws IOException { Files.move( srcFile.toPath(), renameToFile.toPath(), copyOptions ); @@ -264,22 +289,7 @@ public static void copyFile( File srcFile, File dstFile ) throws IOException { //noinspection ResultOfMethodCallIgnored dstFile.getParentFile().mkdirs(); - try ( FileInputStream input = new FileInputStream( srcFile ); - FileOutputStream output = new FileOutputStream( dstFile ) ) - { - int bufferSize = 1024; - byte[] buffer = new byte[bufferSize]; - int bytesRead; - while ( (bytesRead = input.read( buffer )) != -1 ) - { - output.write( buffer, 0, bytesRead ); - } - } - catch ( IOException e ) - { - // Because the message from this cause may not mention which file it's about - throw new IOException( "Could not copy '" + srcFile + "' to '" + dstFile + "'", e ); - } + Files.copy( srcFile.toPath(), dstFile.toPath(), StandardCopyOption.REPLACE_EXISTING ); } public static void copyRecursively( File fromDirectory, File toDirectory ) throws IOException diff --git a/community/io/src/test/java/org/neo4j/adversaries/fs/AdversarialFileSystemAbstraction.java b/community/io/src/test/java/org/neo4j/adversaries/fs/AdversarialFileSystemAbstraction.java index bc02ab2766476..8f5a07a45c585 100644 --- a/community/io/src/test/java/org/neo4j/adversaries/fs/AdversarialFileSystemAbstraction.java +++ b/community/io/src/test/java/org/neo4j/adversaries/fs/AdversarialFileSystemAbstraction.java @@ -185,6 +185,15 @@ public void moveToDirectory( File file, File toDirectory ) throws IOException delegate.moveToDirectory( file, toDirectory ); } + @Override + public void copyToDirectory( File file, File toDirectory ) throws IOException + { + adversary.injectFailure( + SecurityException.class, IllegalArgumentException.class, FileNotFoundException.class, + NullPointerException.class, IOException.class ); + delegate.copyToDirectory( file, toDirectory ); + } + @Override public boolean isDirectory( File file ) { diff --git a/community/io/src/test/java/org/neo4j/graphdb/mockfs/DelegatingFileSystemAbstraction.java b/community/io/src/test/java/org/neo4j/graphdb/mockfs/DelegatingFileSystemAbstraction.java index 12da53dbaa1f8..176855974b3b7 100644 --- a/community/io/src/test/java/org/neo4j/graphdb/mockfs/DelegatingFileSystemAbstraction.java +++ b/community/io/src/test/java/org/neo4j/graphdb/mockfs/DelegatingFileSystemAbstraction.java @@ -65,6 +65,12 @@ public void moveToDirectory( File file, File toDirectory ) throws IOException delegate.moveToDirectory( file, toDirectory ); } + @Override + public void copyToDirectory( File file, File toDirectory ) throws IOException + { + delegate.copyToDirectory( file, toDirectory ); + } + @Override public boolean mkdir( File fileName ) { diff --git a/community/io/src/test/java/org/neo4j/graphdb/mockfs/EphemeralFileSystemAbstraction.java b/community/io/src/test/java/org/neo4j/graphdb/mockfs/EphemeralFileSystemAbstraction.java index 8d1a216df3a7f..447ff5f94e914 100644 --- a/community/io/src/test/java/org/neo4j/graphdb/mockfs/EphemeralFileSystemAbstraction.java +++ b/community/io/src/test/java/org/neo4j/graphdb/mockfs/EphemeralFileSystemAbstraction.java @@ -540,6 +540,13 @@ public void moveToDirectory( File file, File toDirectory ) throws IOException } } + @Override + public void copyToDirectory( File file, File toDirectory ) throws IOException + { + File targetFile = new File( toDirectory, file.getName() ); + copyFile( file, targetFile ); + } + @Override public void copyFile( File from, File to ) throws IOException { @@ -626,6 +633,7 @@ private void copyFile( File from, FileSystemAbstraction fromFs, File to, ByteBuf try ( StoreChannel source = fromFs.open( from, OpenMode.READ ); StoreChannel sink = this.open( to, OpenMode.READ_WRITE ) ) { + sink.truncate( 0 ); for ( int available; (available = (int) (source.size() - source.position())) > 0; ) { buffer.clear(); diff --git a/community/io/src/test/java/org/neo4j/graphdb/mockfs/SelectiveFileSystemAbstraction.java b/community/io/src/test/java/org/neo4j/graphdb/mockfs/SelectiveFileSystemAbstraction.java index bdf90c0b1a6cf..1155f96a1a806 100644 --- a/community/io/src/test/java/org/neo4j/graphdb/mockfs/SelectiveFileSystemAbstraction.java +++ b/community/io/src/test/java/org/neo4j/graphdb/mockfs/SelectiveFileSystemAbstraction.java @@ -167,6 +167,12 @@ public void moveToDirectory( File file, File toDirectory ) throws IOException chooseFileSystem( file ).moveToDirectory( file, toDirectory ); } + @Override + public void copyToDirectory( File file, File toDirectory ) throws IOException + { + chooseFileSystem( file ).copyToDirectory( file, toDirectory ); + } + @Override public void copyFile( File from, File to ) throws IOException { diff --git a/community/io/src/test/java/org/neo4j/io/fs/FileSystemAbstractionTest.java b/community/io/src/test/java/org/neo4j/io/fs/FileSystemAbstractionTest.java index b39d0feccd17e..ad96f2ec392d7 100644 --- a/community/io/src/test/java/org/neo4j/io/fs/FileSystemAbstractionTest.java +++ b/community/io/src/test/java/org/neo4j/io/fs/FileSystemAbstractionTest.java @@ -55,6 +55,7 @@ import static org.hamcrest.Matchers.not; import static org.hamcrest.Matchers.nullValue; import static org.hamcrest.core.Is.is; +import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertSame; @@ -158,25 +159,39 @@ public void moveToDirectoryMustMoveFile() throws Exception } @Test - public void moveToDirectoryMustRecursivelyMoveAllFilesInGivenDirectory() throws Exception + public void copyToDirectoryCopiesFile() throws IOException { File source = new File( path, "source" ); File target = new File( path, "target" ); File file = new File( source, "file" ); - File sourceAfterMove = new File( target, "source" ); - File fileAfterMove = new File( sourceAfterMove, "file" ); + File fileAfterCopy = new File( target, "file" ); fsa.mkdirs( source ); fsa.mkdirs( target ); fsa.create( file ).close(); - assertTrue( fsa.fileExists( source ) ); assertTrue( fsa.fileExists( file ) ); - assertFalse( fsa.fileExists( sourceAfterMove ) ); - assertFalse( fsa.fileExists( fileAfterMove ) ); - fsa.moveToDirectory( source, target ); - assertFalse( fsa.fileExists( source ) ); - assertFalse( fsa.fileExists( file ) ); - assertTrue( fsa.fileExists( sourceAfterMove ) ); - assertTrue( fsa.fileExists( fileAfterMove ) ); + assertFalse( fsa.fileExists( fileAfterCopy ) ); + fsa.copyToDirectory( file, target ); + assertTrue( fsa.fileExists( file ) ); + assertTrue( fsa.fileExists( fileAfterCopy ) ); + } + + @Test + public void copyToDirectoryReplaceExistingFile() throws Exception + { + File source = new File( path, "source" ); + File target = new File( path, "target" ); + File file = new File( source, "file" ); + File targetFile = new File( target, "file" ); + fsa.mkdirs( source ); + fsa.mkdirs( target ); + fsa.create( file ).close(); + + writeIntegerIntoFile( targetFile ); + + fsa.copyToDirectory( file, target ); + assertTrue( fsa.fileExists( file ) ); + assertTrue( fsa.fileExists( targetFile ) ); + assertEquals( 0L, fsa.getFileSize( targetFile ) ); } @Test @@ -812,4 +827,13 @@ private void ensureDirectoryExists( File directory ) throws IOException { fsa.mkdirs( directory ); } + + private void writeIntegerIntoFile( File targetFile ) throws IOException + { + StoreChannel storeChannel = fsa.create( targetFile ); + ByteBuffer byteBuffer = ByteBuffer.allocate( Integer.SIZE ).putInt( 7 ); + byteBuffer.flip(); + storeChannel.writeAll( byteBuffer ); + storeChannel.close(); + } } diff --git a/community/io/src/test/java/org/neo4j/test/rule/fs/FileSystemRule.java b/community/io/src/test/java/org/neo4j/test/rule/fs/FileSystemRule.java index 9aeac9f09f3de..680f473f3c241 100644 --- a/community/io/src/test/java/org/neo4j/test/rule/fs/FileSystemRule.java +++ b/community/io/src/test/java/org/neo4j/test/rule/fs/FileSystemRule.java @@ -185,6 +185,12 @@ public void moveToDirectory( File file, File toDirectory ) throws IOException fs.moveToDirectory( file, toDirectory ); } + @Override + public void copyToDirectory( File file, File toDirectory ) throws IOException + { + fs.copyToDirectory( file, toDirectory ); + } + @Override public void copyFile( File from, File to ) throws IOException { diff --git a/community/kernel/src/main/java/org/neo4j/graphdb/factory/GraphDatabaseSettings.java b/community/kernel/src/main/java/org/neo4j/graphdb/factory/GraphDatabaseSettings.java index c11ed027dc02d..a1260a21d34a4 100644 --- a/community/kernel/src/main/java/org/neo4j/graphdb/factory/GraphDatabaseSettings.java +++ b/community/kernel/src/main/java/org/neo4j/graphdb/factory/GraphDatabaseSettings.java @@ -101,6 +101,19 @@ public class GraphDatabaseSettings implements LoadableConfig public static final Setting neo4j_home = setting( "unsupported.dbms.directories.neo4j_home", PATH, NO_DEFAULT ); + @Description( "Name of the database to load" ) + public static final Setting active_database = setting( "dbms.active_database", STRING, "graph.db" ); + + @Description( "Path of the data directory. You must not configure more than one Neo4j installation to use the " + + "same data directory." ) + public static final Setting data_directory = pathSetting( "dbms.directories.data", "data" ); + + @Internal + public static final Setting database_path = derivedSetting( "unsupported.dbms.directories.database", + data_directory, active_database, + ( data, current ) -> new File( new File( data, "databases" ), current ), + PATH ); + @Title( "Read only database" ) @Description( "Only allow read operations from this Neo4j instance. " + "This mode still requires write access to the directory for lock purposes." ) @@ -433,6 +446,10 @@ public class GraphDatabaseSettings implements LoadableConfig public static final Setting enable_native_schema_index = setting( "unsupported.dbms.enable_native_schema_index", BOOLEAN, TRUE ); + @Description( "Location where Neo4j keeps the logical transaction logs." ) + public static final Setting logical_logs_location = + pathSetting( "dbms.directories.tx_log", "", database_path ); + // Store settings @Description( "Make Neo4j keep the logical transaction logs for being able to backup the database. " + "Can be used for specifying the threshold to prune logical logs after. For example \"10 days\" will " + diff --git a/community/kernel/src/main/java/org/neo4j/kernel/NeoStoreDataSource.java b/community/kernel/src/main/java/org/neo4j/kernel/NeoStoreDataSource.java index bc73d9c7254ae..3a8fb5d2f8e8c 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/NeoStoreDataSource.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/NeoStoreDataSource.java @@ -401,10 +401,10 @@ public void start() throws IOException final LogEntryReader logEntryReader = new VersionAwareLogEntryReader<>(); LogFiles logFiles = LogFilesBuilder.builder(storeDir, fs) - .withLogEntryReader( logEntryReader ) - .withLogFileMonitor( physicalLogMonitor ) - .withConfig( config ) - .withDependencies( dependencies ).build(); + .withLogEntryReader( logEntryReader ) + .withLogFileMonitor( physicalLogMonitor ) + .withConfig( config ) + .withDependencies( dependencies ).build(); LogTailScanner tailScanner = new LogTailScanner( logFiles, logEntryReader, monitors, failOnCorruptedLogFiles ); monitors.addMonitorListener( new LoggingLogTailScannerMonitor( logService.getInternalLog( LogTailScanner.class ) ) ); @@ -450,6 +450,7 @@ public void start() throws IOException PropertyAccessor propertyAccessor = dependencies.resolveDependency( PropertyAccessor.class ); final NeoStoreKernelModule kernelModule = buildKernel( + logFiles, transactionLogModule.transactionAppender(), dependencies.resolveDependency( IndexingService.class ), storageEngine.storeReadLayer(), @@ -647,16 +648,10 @@ private void buildRecovery( life.add( recovery ); } - private NeoStoreKernelModule buildKernel( TransactionAppender appender, - IndexingService indexingService, - StoreReadLayer storeLayer, - DatabaseSchemaState databaseSchemaState, LabelScanStore labelScanStore, - StorageEngine storageEngine, - IndexConfigStore indexConfigStore, - TransactionIdStore transactionIdStore, - AvailabilityGuard availabilityGuard, - SystemNanoClock clock, - PropertyAccessor propertyAccessor ) throws KernelException, IOException + private NeoStoreKernelModule buildKernel( LogFiles logFiles, TransactionAppender appender, IndexingService indexingService, + StoreReadLayer storeLayer, DatabaseSchemaState databaseSchemaState, LabelScanStore labelScanStore, + StorageEngine storageEngine, IndexConfigStore indexConfigStore, TransactionIdStore transactionIdStore, + AvailabilityGuard availabilityGuard, SystemNanoClock clock, PropertyAccessor propertyAccessor ) throws KernelException, IOException { CpuClock cpuClock = CpuClock.NOT_AVAILABLE; if ( config.get( GraphDatabaseSettings.track_query_cpu_time ) ) @@ -702,8 +697,8 @@ private NeoStoreKernelModule buildKernel( TransactionAppender appender, kernel.registerTransactionHook( transactionEventHandlers ); - final NeoStoreFileListing fileListing = new NeoStoreFileListing( storeDir, labelScanStore, indexingService, - explicitIndexProviderLookup, storageEngine ); + final NeoStoreFileListing fileListing = new NeoStoreFileListing( storeDir, logFiles, labelScanStore, + indexingService, explicitIndexProviderLookup, storageEngine ); return new NeoStoreKernelModule( transactionCommitProcess, kernel, kernelTransactions, fileListing ); } diff --git a/community/kernel/src/main/java/org/neo4j/kernel/configuration/Settings.java b/community/kernel/src/main/java/org/neo4j/kernel/configuration/Settings.java index 869e3baa18a25..309e8ad66f3c4 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/configuration/Settings.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/configuration/Settings.java @@ -55,6 +55,7 @@ import static java.lang.Float.parseFloat; import static java.lang.Integer.parseInt; import static java.lang.Long.parseLong; +import static java.lang.String.format; import static org.neo4j.graphdb.factory.GraphDatabaseSettings.default_advertised_address; import static org.neo4j.graphdb.factory.GraphDatabaseSettings.default_listen_address; import static org.neo4j.io.fs.FileUtils.fixSeparatorsInPath; @@ -332,58 +333,12 @@ public String valueDescription() public static Setting pathSetting( String name, String defaultValue ) { - return new ScopeAwareSetting() - { - @Override - protected String provideName() - { - return name; - } - - @Override - public String getDefaultValue() - { - return defaultValue; - } - - @Override - public File from( Configuration config ) - { - return config.get( this ); - } - - @Override - public File apply( Function config ) - { - String value = config.apply( name() ); - if ( value == null ) - { - value = defaultValue; - } - if ( value == null ) - { - return null; - } - - String setting = fixSeparatorsInPath( value ); - File settingFile = new File( setting ); - - if ( settingFile.isAbsolute() ) - { - return settingFile; - } - else - { - return new File( GraphDatabaseSettings.neo4j_home.apply( config ), setting ); - } - } + return new FileSetting( name, defaultValue ); + } - @Override - public String valueDescription() - { - return "A filesystem path; relative paths are resolved against the installation root, __"; - } - }; + public static Setting pathSetting( String name, String defaultValue, Setting relativeRoot ) + { + return new FileSetting( name, defaultValue, relativeRoot ); } private static BiFunction, String> inheritedValue( @@ -734,7 +689,7 @@ else if ( mem.endsWith( "g" ) ) } catch ( NumberFormatException e ) { - throw new IllegalArgumentException( String.format( "%s is not a valid size, must be e.g. 10, 5K, 1M, " + + throw new IllegalArgumentException( format( "%s is not a valid size, must be e.g. 10, 5K, 1M, " + "11G", value ) ); } } @@ -935,7 +890,7 @@ public String apply( String value, Function settings ) @Override public String toString() { - return String.format( MATCHES_PATTERN_MESSAGE, regex ); + return format( MATCHES_PATTERN_MESSAGE, regex ); } }; } @@ -983,7 +938,7 @@ public List apply( List values, Function settings @Override public String toString() { - return String.format( MATCHES_PATTERN_MESSAGE, regex ); + return format( MATCHES_PATTERN_MESSAGE, regex ); } }; } @@ -997,7 +952,7 @@ public T apply( T value, Function settings ) { if ( value != null && value.compareTo( min ) < 0 ) { - throw new IllegalArgumentException( String.format( "minimum allowed value is: %s", min ) ); + throw new IllegalArgumentException( format( "minimum allowed value is: %s", min ) ); } return value; } @@ -1019,7 +974,7 @@ public T apply( T value, Function settings ) { if ( value != null && value.compareTo( max ) > 0 ) { - throw new IllegalArgumentException( String.format( "maximum allowed value is: %s", max ) ); + throw new IllegalArgumentException( format( "maximum allowed value is: %s", max ) ); } return value; } @@ -1045,7 +1000,7 @@ public T apply( T from1, Function from2 ) @Override public String toString() { - return String.format( "is in the range `%s` to `%s`", min, max ); + return format( "is in the range `%s` to `%s`", min, max ); } }; } @@ -1076,7 +1031,7 @@ public String toString() { String description = message; if ( valueFunction != null - && !String.format( MATCHES_PATTERN_MESSAGE, ANY ).equals( + && !format( MATCHES_PATTERN_MESSAGE, ANY ).equals( valueFunction.toString() ) ) { description += " (" + valueFunction.toString() + ")"; @@ -1311,7 +1266,7 @@ public T apply( Function settings ) } catch ( Exception e ) { - throw new IllegalArgumentException( String.format( "Missing mandatory setting '%s'", name() ) ); + throw new IllegalArgumentException( format( "Missing mandatory setting '%s'", name() ) ); } } @@ -1367,4 +1322,73 @@ public String valueDescription() return builder.toString(); } } + + private static class FileSetting extends ScopeAwareSetting + { + private final String name; + private final String defaultValue; + private final Setting relativeRoot; + + FileSetting( String name, String defaultValue ) + { + this( name, defaultValue, GraphDatabaseSettings.neo4j_home ); + } + + FileSetting( String name, String defaultValue, Setting relativeRoot ) + { + this.name = name; + this.defaultValue = defaultValue; + this.relativeRoot = relativeRoot; + } + + @Override + protected String provideName() + { + return name; + } + + @Override + public String getDefaultValue() + { + return defaultValue; + } + + @Override + public File from( Configuration config ) + { + return config.get( this ); + } + + @Override + public File apply( Function config ) + { + String value = config.apply( name() ); + if ( value == null ) + { + value = defaultValue; + } + if ( value == null ) + { + return null; + } + + String setting = fixSeparatorsInPath( value ); + File settingFile = new File( setting ); + + if ( settingFile.isAbsolute() ) + { + return settingFile; + } + else + { + return new File( relativeRoot.apply( config ), setting ); + } + } + + @Override + public String valueDescription() + { + return "A filesystem path; relative paths are resolved against the root, _<" + relativeRoot.name() + ">_"; + } + } } diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/factory/PlatformModule.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/factory/PlatformModule.java index 4cf27a98c0e18..af15ad0436ef4 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/factory/PlatformModule.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/factory/PlatformModule.java @@ -179,10 +179,6 @@ public PlatformModule( File providedStoreDir, Config config, DatabaseInfo databa diagnosticsManager = life.add( dependencies .satisfyDependency( new DiagnosticsManager( logging.getInternalLog( DiagnosticsManager.class ) ) ) ); - // TODO please fix the bad dependencies instead of doing this. - // this was the place of the XaDataSourceManager. NeoStoreXaDataSource is create further down than - // (specifically) KernelExtensions, which creates an interesting out-of-order issue with #doAfterRecovery(). - // Anyways please fix this. dependencies.satisfyDependency( dataSourceManager ); availabilityGuard = dependencies.satisfyDependency( createAvailabilityGuard() ); diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/recovery/RecoveryRequiredChecker.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/recovery/RecoveryRequiredChecker.java index b398bbf67f227..a14fc8728fe47 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/recovery/RecoveryRequiredChecker.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/recovery/RecoveryRequiredChecker.java @@ -24,6 +24,7 @@ import org.neo4j.io.fs.FileSystemAbstraction; import org.neo4j.io.pagecache.PageCache; +import org.neo4j.kernel.configuration.Config; import org.neo4j.kernel.impl.store.NeoStores; import org.neo4j.kernel.impl.transaction.log.ReadableClosablePositionAwareChannel; import org.neo4j.kernel.impl.transaction.log.entry.LogEntryReader; @@ -44,11 +45,13 @@ public class RecoveryRequiredChecker private final FileSystemAbstraction fs; private final PageCache pageCache; private final Monitors monitors; + private Config config; - public RecoveryRequiredChecker( FileSystemAbstraction fs, PageCache pageCache, Monitors monitors ) + public RecoveryRequiredChecker( FileSystemAbstraction fs, PageCache pageCache, Config config, Monitors monitors ) { this.fs = fs; this.pageCache = pageCache; + this.config = config; this.monitors = monitors; } @@ -62,7 +65,9 @@ public boolean isRecoveryRequiredAt( File dataDir ) throws IOException } LogEntryReader reader = new VersionAwareLogEntryReader<>(); - LogFiles logFiles = LogFilesBuilder.activeFilesBuilder( dataDir, fs, pageCache ).withLogEntryReader( reader ).build(); + LogFiles logFiles = LogFilesBuilder.activeFilesBuilder( dataDir, fs, pageCache ) + .withConfig( config ) + .withLogEntryReader( reader ).build(); LogTailScanner tailScanner = new LogTailScanner( logFiles, reader, monitors ); return new RecoveryStartInformationProvider( tailScanner, NO_MONITOR ).get().isRecoveryRequired(); } diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/storemigration/participant/StoreMigrator.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/storemigration/participant/StoreMigrator.java index dbb4516f4ccd9..32e4d64398b7b 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/storemigration/participant/StoreMigrator.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/storemigration/participant/StoreMigrator.java @@ -309,7 +309,7 @@ private TransactionId specificTransactionInformationSupplier( long lastTransacti : new TransactionId( lastTransactionId, UNKNOWN_TX_CHECKSUM, UNKNOWN_TX_COMMIT_TIMESTAMP ); } - private LogPosition extractTransactionLogPosition( File neoStore, File storeDir, long lastTxId ) throws IOException + LogPosition extractTransactionLogPosition( File neoStore, File storeDir, long lastTxId ) throws IOException { long lastClosedTxLogVersion = MetaDataStore.getRecord( pageCache, neoStore, Position.LAST_CLOSED_TRANSACTION_LOG_VERSION ); @@ -327,7 +327,9 @@ private LogPosition extractTransactionLogPosition( File neoStore, File storeDir, return new LogPosition( BASE_TX_LOG_VERSION, BASE_TX_LOG_BYTE_OFFSET ); } - LogFiles logFiles = LogFilesBuilder.activeFilesBuilder( storeDir,fileSystem, pageCache ).build(); + LogFiles logFiles = LogFilesBuilder.activeFilesBuilder( storeDir,fileSystem, pageCache ) + .withConfig( config ) + .build(); long logVersion = logFiles.getHighestLogVersion(); if ( logVersion == -1 ) { diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/transaction/log/ReadOnlyTransactionStore.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/transaction/log/ReadOnlyTransactionStore.java index e9e3353384cf6..cf51029ff02b0 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/transaction/log/ReadOnlyTransactionStore.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/transaction/log/ReadOnlyTransactionStore.java @@ -24,6 +24,7 @@ import org.neo4j.io.fs.FileSystemAbstraction; import org.neo4j.io.pagecache.PageCache; +import org.neo4j.kernel.configuration.Config; import org.neo4j.kernel.impl.transaction.log.TransactionMetadataCache.TransactionMetadata; import org.neo4j.kernel.impl.transaction.log.entry.LogEntryReader; import org.neo4j.kernel.impl.transaction.log.entry.VersionAwareLogEntryReader; @@ -41,13 +42,14 @@ public class ReadOnlyTransactionStore implements Lifecycle, LogicalTransactionSt private final LifeSupport life = new LifeSupport(); private final LogicalTransactionStore physicalStore; - public ReadOnlyTransactionStore( PageCache pageCache, FileSystemAbstraction fs, File fromPath, Monitors monitors ) - throws IOException + public ReadOnlyTransactionStore( PageCache pageCache, FileSystemAbstraction fs, File fromPath, Config config, + Monitors monitors ) throws IOException { TransactionMetadataCache transactionMetadataCache = new TransactionMetadataCache( 100 ); LogEntryReader logEntryReader = new VersionAwareLogEntryReader<>(); LogFiles logFiles = LogFilesBuilder .activeFilesBuilder( fromPath, fs, pageCache ).withLogEntryReader( logEntryReader ) + .withConfig( config ) .build(); physicalStore = new PhysicalLogicalTransactionStore( logFiles, transactionMetadataCache, logEntryReader, monitors, true ); diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/transaction/log/files/LogFiles.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/transaction/log/files/LogFiles.java index 56c8ac6175285..60eca881cb829 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/transaction/log/files/LogFiles.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/transaction/log/files/LogFiles.java @@ -39,6 +39,10 @@ public interface LogFiles extends Lifecycle File[] logFiles(); + boolean isLogFile( File file ); + + File logFilesDirectory(); + File getLogFileForVersion( long version ); File getHighestLogFile(); diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/transaction/log/files/LogFilesBuilder.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/transaction/log/files/LogFilesBuilder.java index 8880905b58707..f541f0e9f252f 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/transaction/log/files/LogFilesBuilder.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/transaction/log/files/LogFilesBuilder.java @@ -21,9 +21,11 @@ import java.io.File; import java.io.IOException; +import java.nio.file.Path; import java.util.function.LongSupplier; import java.util.function.Supplier; +import org.neo4j.graphdb.factory.GraphDatabaseSettings; import org.neo4j.io.fs.FileSystemAbstraction; import org.neo4j.io.pagecache.PageCache; import org.neo4j.kernel.configuration.Config; @@ -36,6 +38,7 @@ import org.neo4j.kernel.impl.util.Dependencies; import static java.util.Objects.requireNonNull; +import static org.neo4j.graphdb.factory.GraphDatabaseSettings.database_path; import static org.neo4j.graphdb.factory.GraphDatabaseSettings.logical_log_rotation_threshold; /** @@ -55,6 +58,7 @@ public class LogFilesBuilder private boolean readOnly; private PageCache pageCache; private File storeDirectory; + private File logsDirectory; private Config config; private Long rotationThreshold; private LogEntryReader logEntryReader; @@ -100,13 +104,15 @@ public static LogFilesBuilder activeFilesBuilder( File storeDirectory, FileSyste /** * Build log files that will be able to perform only operations on a lgo files directly. * Any operation that will require access to a store or other parts of runtime will fail. - * Should be mainly used only for testing purposes. - * @param storeDirectory store directory + * Should be mainly used only for testing purposes or when only file based operations will be performed + * @param logsDirectory log files directory * @param fileSystem file system */ - public static LogFilesBuilder logFilesBasedOnlyBuilder( File storeDirectory, FileSystemAbstraction fileSystem ) + public static LogFilesBuilder logFilesBasedOnlyBuilder( File logsDirectory, FileSystemAbstraction fileSystem ) { - LogFilesBuilder builder = builder( storeDirectory, fileSystem ); + LogFilesBuilder builder = new LogFilesBuilder(); + builder.logsDirectory = logsDirectory; + builder.fileSystem = fileSystem; builder.fileBasedOperationsOnly = true; return builder; } @@ -168,7 +174,38 @@ public LogFilesBuilder withDependencies( Dependencies dependencies ) public LogFiles build() throws IOException { TransactionLogFilesContext filesContext = buildContext(); - return new TransactionLogFiles( storeDirectory, logFileName, filesContext ); + File logsDirectory = getLogsDirectory(); + filesContext.getFileSystem().mkdirs( logsDirectory ); + return new TransactionLogFiles( logsDirectory, logFileName, filesContext ); + } + + private File getLogsDirectory() + { + if ( logsDirectory != null ) + { + return logsDirectory; + } + if ( config != null ) + { + File neo4jHome = config.get( GraphDatabaseSettings.neo4j_home ); + File databasePath = config.get( database_path ); + File logicalLogsLocation = config.get( GraphDatabaseSettings.logical_logs_location ); + if ( storeDirectory.equals( neo4jHome ) && databasePath.equals( logicalLogsLocation ) ) + { + return storeDirectory; + } + if ( logicalLogsLocation.isAbsolute() ) + { + return logicalLogsLocation; + } + if ( neo4jHome == null || !storeDirectory.equals( databasePath ) ) + { + Path relativeLogicalLogPath = databasePath.toPath().relativize( logicalLogsLocation.toPath() ); + return new File( storeDirectory, relativeLogicalLogPath.toString() ); + } + return logicalLogsLocation; + } + return storeDirectory; } TransactionLogFilesContext buildContext() throws IOException diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/transaction/log/files/TransactionLogFiles.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/transaction/log/files/TransactionLogFiles.java index 801708434aa1a..b2d05819d0f9a 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/transaction/log/files/TransactionLogFiles.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/transaction/log/files/TransactionLogFiles.java @@ -51,6 +51,7 @@ public class TransactionLogFiles extends LifecycleAdapter implements LogFiles { public static final String DEFAULT_NAME = "neostore.transaction.db"; public static final FilenameFilter DEFAULT_FILENAME_FILTER = TransactionLogFilesHelper.DEFAULT_FILENAME_FILTER; + private static final File[] EMPTY_FILES_ARRAY = {}; private final TransactionLogFilesContext logFilesContext; private final TransactionLogFileInformation logFileInformation; @@ -60,11 +61,13 @@ public class TransactionLogFiles extends LifecycleAdapter implements LogFiles private final LogFileCreationMonitor monitor; private final TransactionLogFilesHelper fileHelper; private final TransactionLogFile logFile; + private final File logsDirectory; - TransactionLogFiles( File directory, String name, TransactionLogFilesContext context ) + TransactionLogFiles( File logsDirectory, String name, TransactionLogFilesContext context ) { this.logFilesContext = context; - this.fileHelper = new TransactionLogFilesHelper( directory, name ); + this.logsDirectory = logsDirectory; + this.fileHelper = new TransactionLogFilesHelper( logsDirectory, name ); this.fileSystem = context.getFileSystem(); this.monitor = context.getLogFileCreationMonitor(); this.logHeaderCache = new LogHeaderCache( 1000 ); @@ -105,7 +108,24 @@ public long getLogVersion( String historyLogFilename ) @Override public File[] logFiles() { - return fileSystem.listFiles( fileHelper.getParentDirectory(), fileHelper.getLogFilenameFilter() ); + File[] files = fileSystem.listFiles( fileHelper.getParentDirectory(), fileHelper.getLogFilenameFilter() ); + if ( files == null ) + { + return EMPTY_FILES_ARRAY; + } + return files; + } + + @Override + public boolean isLogFile( File file ) + { + return fileHelper.getLogFilenameFilter().accept( null, file.getName() ); + } + + @Override + public File logFilesDirectory() + { + return logsDirectory; } @Override diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/transaction/log/files/TransactionLogFilesHelper.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/transaction/log/files/TransactionLogFilesHelper.java index bb37bb7c2c9c7..9627f23e87f02 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/transaction/log/files/TransactionLogFilesHelper.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/transaction/log/files/TransactionLogFilesHelper.java @@ -31,7 +31,7 @@ class TransactionLogFilesHelper private static final String VERSION_SUFFIX = "."; private static final String REGEX_VERSION_SUFFIX = "\\."; - public static final FilenameFilter DEFAULT_FILENAME_FILTER = new LogicalLogFilenameFilter( REGEX_DEFAULT_NAME ); + static final FilenameFilter DEFAULT_FILENAME_FILTER = new LogicalLogFilenameFilter( REGEX_DEFAULT_NAME ); private final File logBaseName; private final FilenameFilter logFileFilter; diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/transaction/state/NeoStoreFileListing.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/transaction/state/NeoStoreFileListing.java index d27476f2bde9f..79246a28438a4 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/transaction/state/NeoStoreFileListing.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/transaction/state/NeoStoreFileListing.java @@ -34,9 +34,9 @@ import org.neo4j.kernel.impl.api.ExplicitIndexProviderLookup; import org.neo4j.kernel.impl.api.index.IndexingService; import org.neo4j.kernel.impl.index.IndexConfigStore; -import org.neo4j.kernel.impl.store.MetaDataStore; import org.neo4j.kernel.impl.store.StoreType; import org.neo4j.kernel.impl.store.format.RecordFormat; +import org.neo4j.kernel.impl.transaction.log.files.LogFiles; import org.neo4j.kernel.spi.explicitindex.IndexImplementation; import org.neo4j.storageengine.api.StorageEngine; import org.neo4j.storageengine.api.StoreFileMetadata; @@ -47,17 +47,22 @@ public class NeoStoreFileListing { private final File storeDir; + private final LogFiles logFiles; private final LabelScanStore labelScanStore; private final IndexingService indexingService; private final ExplicitIndexProviderLookup explicitIndexProviders; private final StorageEngine storageEngine; private final Function toNotAStoreTypeFile = file -> new StoreFileMetadata( file, RecordFormat.NO_RECORD_SIZE ); + private final Function logFileMapper = + file -> new StoreFileMetadata( file, RecordFormat.NO_RECORD_SIZE, true ); - public NeoStoreFileListing( File storeDir, LabelScanStore labelScanStore, IndexingService indexingService, + public NeoStoreFileListing( File storeDir, LogFiles logFiles, + LabelScanStore labelScanStore, IndexingService indexingService, ExplicitIndexProviderLookup explicitIndexProviders, StorageEngine storageEngine ) { this.storeDir = storeDir; + this.logFiles = logFiles; this.labelScanStore = labelScanStore; this.indexingService = indexingService; this.explicitIndexProviders = explicitIndexProviders; @@ -100,20 +105,20 @@ private void placeMetaDataStoreLast( List files ) private void gatherNonRecordStores( Collection files, boolean includeLogs ) { - final File[] storeFiles = storeDir.listFiles(); - if ( storeFiles == null ) + File[] indexFiles = storeDir.listFiles( ( dir, name ) -> name.equals( IndexConfigStore.INDEX_DB_FILE_NAME ) ); + if ( indexFiles != null ) { - return; - } - for ( File file : storeFiles ) - { - if ( file.getName().equals( IndexConfigStore.INDEX_DB_FILE_NAME ) ) + for ( File file : indexFiles ) { files.add( toNotAStoreTypeFile.apply( file ) ); } - else if ( includeLogs && transactionLogFile( file.getName() ) ) + } + if ( includeLogs ) + { + File[] logFiles = this.logFiles.logFiles(); + for ( File logFile : logFiles ) { - files.add( toNotAStoreTypeFile.apply( file ) ); + files.add( logFileMapper.apply( logFile ) ); } } } @@ -155,11 +160,6 @@ private void gatherNeoStoreFiles( final Collection targetFile targetFiles.addAll( storageEngine.listStorageFiles() ); } - private boolean transactionLogFile( String name ) - { - return name.startsWith( MetaDataStore.DEFAULT_NAME + ".transaction" ) && !name.endsWith( ".active" ); - } - private static final class MultiResource implements Resource { private final Collection snapshots; diff --git a/community/kernel/src/main/java/org/neo4j/storageengine/api/StoreFileMetadata.java b/community/kernel/src/main/java/org/neo4j/storageengine/api/StoreFileMetadata.java index 78392c5f84890..1dd5a07b05616 100644 --- a/community/kernel/src/main/java/org/neo4j/storageengine/api/StoreFileMetadata.java +++ b/community/kernel/src/main/java/org/neo4j/storageengine/api/StoreFileMetadata.java @@ -25,11 +25,18 @@ public class StoreFileMetadata { private final File file; private final int recordSize; + private final boolean isLogFile; public StoreFileMetadata( File file, int recordSize ) + { + this( file, recordSize, false ); + } + + public StoreFileMetadata( File file, int recordSize, boolean isLogFile ) { this.file = file; this.recordSize = recordSize; + this.isLogFile = isLogFile; } public File file() @@ -41,4 +48,9 @@ public int recordSize() { return recordSize; } + + public boolean isLogFile() + { + return isLogFile; + } } diff --git a/community/kernel/src/test/java/org/neo4j/graphdb/TransactionLogsInSeparateLocationIT.java b/community/kernel/src/test/java/org/neo4j/graphdb/TransactionLogsInSeparateLocationIT.java new file mode 100644 index 0000000000000..93c70428afcfc --- /dev/null +++ b/community/kernel/src/test/java/org/neo4j/graphdb/TransactionLogsInSeparateLocationIT.java @@ -0,0 +1,100 @@ +/* + * Copyright (c) 2002-2017 "Neo Technology," + * Network Engine for Objects in Lund AB [http://neotechnology.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.graphdb; + +import org.junit.Rule; +import org.junit.Test; + +import java.io.File; +import java.io.IOException; + +import org.neo4j.graphdb.factory.GraphDatabaseSettings; +import org.neo4j.io.fs.FileSystemAbstraction; +import org.neo4j.kernel.impl.transaction.log.PhysicalLogVersionedStoreChannel; +import org.neo4j.kernel.impl.transaction.log.files.LogFiles; +import org.neo4j.kernel.impl.transaction.log.files.LogFilesBuilder; +import org.neo4j.test.TestGraphDatabaseFactory; +import org.neo4j.test.rule.TestDirectory; +import org.neo4j.test.rule.fs.DefaultFileSystemRule; +import org.neo4j.test.rule.fs.FileSystemRule; + +import static org.hamcrest.Matchers.greaterThan; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertTrue; + +public class TransactionLogsInSeparateLocationIT +{ + @Rule + public final TestDirectory testDirectory = TestDirectory.testDirectory(); + @Rule + public final FileSystemRule fileSystemRule = new DefaultFileSystemRule(); + + @Test + public void databaseWithTransactionLogsInSeparateRelativeLocation() throws IOException + { + File storeDir = testDirectory.graphDbDir(); + File txDirectory = new File( storeDir, "transaction-logs" ); + performTransactions( txDirectory.getName(), storeDir ); + verifyTransactionLogs( txDirectory, storeDir ); + } + + @Test + public void databaseWithTransactionLogsInSeparateAbsoluteLocation() throws IOException + { + File storeDir = testDirectory.graphDbDir(); + File txDirectory = testDirectory.directory( "transaction-logs" ); + performTransactions( txDirectory.getAbsolutePath(), storeDir ); + verifyTransactionLogs( txDirectory, storeDir ); + } + + private void performTransactions( String txPath, File storeDir ) throws IOException + { + GraphDatabaseService database = new TestGraphDatabaseFactory().newEmbeddedDatabaseBuilder( storeDir ) + .setConfig( GraphDatabaseSettings.logical_logs_location, txPath ) + .newGraphDatabase(); + for ( int i = 0; i < 10; i++ ) + { + try ( Transaction transaction = database.beginTx() ) + { + Node node = database.createNode(); + node.setProperty( "a", "b" ); + node.setProperty( "c", "d" ); + transaction.success(); + } + } + database.shutdown(); + } + + private void verifyTransactionLogs( File txDirectory, File storeDir ) throws IOException + { + FileSystemAbstraction fileSystem = fileSystemRule.get(); + LogFiles storeDirLogs = LogFilesBuilder.logFilesBasedOnlyBuilder( storeDir, fileSystem ).build(); + assertFalse( storeDirLogs.versionExists( 0 ) ); + + LogFiles txDirectoryLogs = LogFilesBuilder.logFilesBasedOnlyBuilder( txDirectory, fileSystem ).build(); + assertTrue( txDirectoryLogs.versionExists( 0 ) ); + try ( PhysicalLogVersionedStoreChannel physicalLogVersionedStoreChannel = txDirectoryLogs.openForVersion( 0 ) ) + { + assertThat( physicalLogVersionedStoreChannel.size(), greaterThan( 0L ) ); + } + } + +} diff --git a/community/kernel/src/test/java/org/neo4j/kernel/impl/recovery/RecoveryRequiredCheckerTest.java b/community/kernel/src/test/java/org/neo4j/kernel/impl/recovery/RecoveryRequiredCheckerTest.java index b1579267c7ace..7faf0d9cbab98 100644 --- a/community/kernel/src/test/java/org/neo4j/kernel/impl/recovery/RecoveryRequiredCheckerTest.java +++ b/community/kernel/src/test/java/org/neo4j/kernel/impl/recovery/RecoveryRequiredCheckerTest.java @@ -32,6 +32,7 @@ import org.neo4j.graphdb.mockfs.EphemeralFileSystemAbstraction; import org.neo4j.io.fs.FileSystemAbstraction; import org.neo4j.io.pagecache.PageCache; +import org.neo4j.kernel.configuration.Config; import org.neo4j.kernel.monitoring.Monitors; import org.neo4j.test.TestGraphDatabaseFactory; import org.neo4j.test.rule.PageCacheRule; @@ -40,6 +41,7 @@ import static org.hamcrest.CoreMatchers.is; import static org.junit.Assert.assertThat; +import static org.neo4j.graphdb.factory.GraphDatabaseSettings.logical_logs_location; public class RecoveryRequiredCheckerTest { @@ -66,7 +68,7 @@ public void setup() public void shouldNotWantToRecoverIntactStore() throws Exception { PageCache pageCache = pageCacheRule.getPageCache( fileSystem ); - RecoveryRequiredChecker recoverer = getRecoveryChecker( fileSystem, pageCache ); + RecoveryRequiredChecker recoverer = getRecoveryCheckerWithDefaultConfig( fileSystem, pageCache ); assertThat( recoverer.isRecoveryRequiredAt( storeDir ), is( false ) ); } @@ -74,11 +76,11 @@ public void shouldNotWantToRecoverIntactStore() throws Exception @Test public void shouldWantToRecoverBrokenStore() throws Exception { - try ( FileSystemAbstraction fileSystemAbstraction = createSomeDataAndCrash( storeDir, fileSystem ) ) + try ( FileSystemAbstraction fileSystemAbstraction = createAndCrashWithDefaultConfig() ) { PageCache pageCache = pageCacheRule.getPageCache( fileSystemAbstraction ); - RecoveryRequiredChecker recoverer = getRecoveryChecker( fileSystemAbstraction, pageCache ); + RecoveryRequiredChecker recoverer = getRecoveryCheckerWithDefaultConfig( fileSystemAbstraction, pageCache ); assertThat( recoverer.isRecoveryRequiredAt( storeDir ), is( true ) ); } @@ -87,11 +89,11 @@ public void shouldWantToRecoverBrokenStore() throws Exception @Test public void shouldBeAbleToRecoverBrokenStore() throws Exception { - try ( FileSystemAbstraction fileSystemAbstraction = createSomeDataAndCrash( storeDir, fileSystem ) ) + try ( FileSystemAbstraction fileSystemAbstraction = createAndCrashWithDefaultConfig() ) { PageCache pageCache = pageCacheRule.getPageCache( fileSystemAbstraction ); - RecoveryRequiredChecker recoverer = getRecoveryChecker( fileSystemAbstraction, pageCache ); + RecoveryRequiredChecker recoverer = getRecoveryCheckerWithDefaultConfig( fileSystemAbstraction, pageCache ); assertThat( recoverer.isRecoveryRequiredAt( storeDir ), is( true ) ); @@ -101,16 +103,66 @@ public void shouldBeAbleToRecoverBrokenStore() throws Exception } } - private RecoveryRequiredChecker getRecoveryChecker( FileSystemAbstraction fileSystem, PageCache pageCache ) + @Test + public void shouldBeAbleToRecoverBrokenStoreWithLogsInSeparateRelativeLocation() throws Exception + { + File customTransactionLogsLocation = new File( storeDir, "tx-logs" ); + Config config = Config.defaults( logical_logs_location, customTransactionLogsLocation.getName() ); + recoverBrokenStoreWithConfig( config ); + } + + @Test + public void shouldBeAbleToRecoverBrokenStoreWithLogsInSeparateAbsoluteLocation() throws Exception + { + File customTransactionLogsLocation = testDirectory.directory( "tx-logs" ); + Config config = Config.defaults( logical_logs_location, customTransactionLogsLocation.getAbsolutePath() ); + recoverBrokenStoreWithConfig( config ); + } + + private void recoverBrokenStoreWithConfig( Config config ) throws IOException + { + try ( FileSystemAbstraction fileSystemAbstraction = createSomeDataAndCrash( storeDir, fileSystem, config ) ) + { + PageCache pageCache = pageCacheRule.getPageCache( fileSystemAbstraction ); + + RecoveryRequiredChecker recoverer = getRecoveryChecker( fileSystemAbstraction, pageCache, config ); + + assertThat( recoverer.isRecoveryRequiredAt( storeDir ), is( true ) ); + + new TestGraphDatabaseFactory() + .setFileSystem( fileSystemAbstraction ) + .newEmbeddedDatabaseBuilder( storeDir ) + .setConfig( config.getRaw() ) + .newGraphDatabase() + .shutdown(); + + assertThat( recoverer.isRecoveryRequiredAt( storeDir ), is( false ) ); + } + } + + private FileSystemAbstraction createAndCrashWithDefaultConfig() throws IOException + { + return createSomeDataAndCrash( storeDir, fileSystem, Config.defaults() ); + } + + private RecoveryRequiredChecker getRecoveryCheckerWithDefaultConfig( FileSystemAbstraction fileSystem, PageCache pageCache ) + { + return getRecoveryChecker( fileSystem, pageCache, Config.defaults() ); + } + + private RecoveryRequiredChecker getRecoveryChecker( FileSystemAbstraction fileSystem, PageCache pageCache, Config config ) { - return new RecoveryRequiredChecker( fileSystem, pageCache, monitors ); + return new RecoveryRequiredChecker( fileSystem, pageCache, config, monitors ); } - private FileSystemAbstraction createSomeDataAndCrash( File store, EphemeralFileSystemAbstraction fileSystem ) - throws IOException + private FileSystemAbstraction createSomeDataAndCrash( File store, EphemeralFileSystemAbstraction fileSystem, + Config config ) { - final GraphDatabaseService db = - new TestGraphDatabaseFactory().setFileSystem( fileSystem ).newImpermanentDatabase( store ); + final GraphDatabaseService db = new TestGraphDatabaseFactory() + .setFileSystem( fileSystem ) + .newImpermanentDatabaseBuilder( store ) + .setConfig( config.getRaw() ) + .newGraphDatabase(); try ( Transaction tx = db.beginTx() ) { diff --git a/community/kernel/src/test/java/org/neo4j/kernel/impl/store/TestStoreAccess.java b/community/kernel/src/test/java/org/neo4j/kernel/impl/store/TestStoreAccess.java index a17851f19ede7..75560f63a899d 100644 --- a/community/kernel/src/test/java/org/neo4j/kernel/impl/store/TestStoreAccess.java +++ b/community/kernel/src/test/java/org/neo4j/kernel/impl/store/TestStoreAccess.java @@ -81,7 +81,7 @@ private EphemeralFileSystemAbstraction produceUncleanStore() private boolean isUnclean( FileSystemAbstraction fileSystem ) throws IOException { PageCache pageCache = pageCacheRule.getPageCache( fileSystem ); - - return new RecoveryRequiredChecker( fileSystem, pageCache, monitors ).isRecoveryRequiredAt( storeDir ); + RecoveryRequiredChecker requiredChecker = new RecoveryRequiredChecker( fileSystem, pageCache, Config.defaults(), monitors ); + return requiredChecker.isRecoveryRequiredAt( storeDir ); } } diff --git a/community/kernel/src/test/java/org/neo4j/kernel/impl/storemigration/participant/StoreMigratorTest.java b/community/kernel/src/test/java/org/neo4j/kernel/impl/storemigration/participant/StoreMigratorTest.java index 11fcbe4f185eb..9e9aac4bbc9a1 100644 --- a/community/kernel/src/test/java/org/neo4j/kernel/impl/storemigration/participant/StoreMigratorTest.java +++ b/community/kernel/src/test/java/org/neo4j/kernel/impl/storemigration/participant/StoreMigratorTest.java @@ -27,18 +27,24 @@ import java.io.File; import java.io.IOException; +import org.neo4j.graphdb.GraphDatabaseService; +import org.neo4j.graphdb.Node; +import org.neo4j.graphdb.Transaction; import org.neo4j.io.pagecache.PageCache; import org.neo4j.kernel.configuration.Config; import org.neo4j.kernel.impl.logging.LogService; import org.neo4j.kernel.impl.logging.NullLogService; import org.neo4j.kernel.impl.logging.SimpleLogService; +import org.neo4j.kernel.impl.store.MetaDataStore; import org.neo4j.kernel.impl.store.TransactionId; +import org.neo4j.kernel.impl.store.format.standard.MetaDataRecordFormat; import org.neo4j.kernel.impl.store.format.standard.StandardV3_0; import org.neo4j.kernel.impl.store.format.standard.StandardV3_2; import org.neo4j.kernel.impl.transaction.log.LogPosition; import org.neo4j.kernel.impl.transaction.log.TransactionIdStore; import org.neo4j.kernel.impl.util.monitoring.ProgressReporter; import org.neo4j.logging.NullLogProvider; +import org.neo4j.test.TestGraphDatabaseFactory; import org.neo4j.test.rule.PageCacheRule; import org.neo4j.test.rule.RandomRule; import org.neo4j.test.rule.TestDirectory; @@ -46,7 +52,9 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; import static org.mockito.Mockito.mock; +import static org.neo4j.graphdb.factory.GraphDatabaseSettings.logical_logs_location; import static org.neo4j.kernel.impl.store.MetaDataStore.DEFAULT_NAME; import static org.neo4j.kernel.impl.store.MetaDataStore.Position.LAST_TRANSACTION_CHECKSUM; import static org.neo4j.kernel.impl.store.MetaDataStore.Position.LAST_TRANSACTION_COMMIT_TIMESTAMP; @@ -134,6 +142,51 @@ public void shouldGenerateTransactionInformationWhenLogsNotPresent() throws Exce assertEquals( TransactionIdStore.UNKNOWN_TX_COMMIT_TIMESTAMP, actual.commitTimestamp() ); } + @Test + public void extractTransactionInformationFromLogsInCustomRelativeLocation() throws Exception + { + File storeDir = directory.graphDbDir(); + File customLogLocation = new File( storeDir, "customLogLocation" ); + extractTransactionalInformationFromLogs( customLogLocation.getName(), customLogLocation, storeDir ); + } + + @Test + public void extractTransactionInformationFromLogsInCustomAbsoluteLocation() throws Exception + { + File storeDir = directory.graphDbDir(); + File customLogLocation = directory.directory( "customLogLocation" ); + extractTransactionalInformationFromLogs( customLogLocation.getAbsolutePath(), customLogLocation, storeDir ); + } + + private void extractTransactionalInformationFromLogs( String path, File customLogLocation, File storeDir ) throws IOException + { + LogService logService = new SimpleLogService( NullLogProvider.getInstance(), NullLogProvider.getInstance() ); + File neoStore = new File( storeDir, DEFAULT_NAME ); + + GraphDatabaseService database = new TestGraphDatabaseFactory().newEmbeddedDatabaseBuilder( storeDir ) + .setConfig( logical_logs_location, path ).newGraphDatabase(); + for ( int i = 0; i < 10; i++ ) + { + try ( Transaction transaction = database.beginTx() ) + { + Node node = database.createNode(); + transaction.success(); + } + } + database.shutdown(); + + MetaDataStore.setRecord( pageCache, neoStore, MetaDataStore.Position.LAST_CLOSED_TRANSACTION_LOG_VERSION, + MetaDataRecordFormat.FIELD_NOT_PRESENT ); + Config config = Config.defaults( logical_logs_location, path ); + StoreMigrator migrator = new StoreMigrator( fileSystemRule.get(), pageCache, config, logService ); + LogPosition logPosition = migrator.extractTransactionLogPosition( neoStore, storeDir, 100 ); + + File[] logFiles = customLogLocation.listFiles(); + assertNotNull( logFiles ); + assertEquals( 0, logPosition.getLogVersion() ); + assertEquals( logFiles[0].length(), logPosition.getByteOffset() ); + } + @Test public void shouldGenerateTransactionInformationWhenLogsAreEmpty() throws Exception { diff --git a/community/kernel/src/test/java/org/neo4j/kernel/impl/transaction/log/files/LogFilesBuilderTest.java b/community/kernel/src/test/java/org/neo4j/kernel/impl/transaction/log/files/LogFilesBuilderTest.java index bc8463b6e0cef..35f8c6f3fcfb4 100644 --- a/community/kernel/src/test/java/org/neo4j/kernel/impl/transaction/log/files/LogFilesBuilderTest.java +++ b/community/kernel/src/test/java/org/neo4j/kernel/impl/transaction/log/files/LogFilesBuilderTest.java @@ -26,19 +26,21 @@ import java.io.File; import java.io.IOException; -import org.neo4j.graphdb.mockfs.EphemeralFileSystemAbstraction; import org.neo4j.io.ByteUnit; +import org.neo4j.io.fs.DefaultFileSystemAbstraction; import org.neo4j.io.pagecache.PageCache; +import org.neo4j.kernel.configuration.Config; import org.neo4j.kernel.impl.transaction.SimpleLogVersionRepository; import org.neo4j.kernel.impl.transaction.SimpleTransactionIdStore; import org.neo4j.kernel.impl.util.Dependencies; import org.neo4j.test.rule.PageCacheRule; import org.neo4j.test.rule.TestDirectory; -import org.neo4j.test.rule.fs.EphemeralFileSystemRule; +import org.neo4j.test.rule.fs.DefaultFileSystemRule; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertSame; +import static org.neo4j.graphdb.factory.GraphDatabaseSettings.logical_logs_location; import static org.neo4j.kernel.impl.transaction.log.files.LogFilesBuilder.activeFilesBuilder; import static org.neo4j.kernel.impl.transaction.log.files.LogFilesBuilder.builder; import static org.neo4j.kernel.impl.transaction.log.files.LogFilesBuilder.logFilesBasedOnlyBuilder; @@ -48,17 +50,17 @@ public class LogFilesBuilderTest @Rule public final TestDirectory testDirectory = TestDirectory.testDirectory(); @Rule - public final EphemeralFileSystemRule fileSystemRule = new EphemeralFileSystemRule(); + public final DefaultFileSystemRule fileSystemRule = new DefaultFileSystemRule(); @Rule public final PageCacheRule pageCacheRule = new PageCacheRule(); - private File directory; - private EphemeralFileSystemAbstraction fileSystem; + private File storeDirectory; + private DefaultFileSystemAbstraction fileSystem; @Before public void setUp() throws Exception { - directory = testDirectory.directory(); + storeDirectory = testDirectory.directory(); fileSystem = fileSystemRule.get(); } @@ -66,7 +68,7 @@ public void setUp() throws Exception public void buildActiveFilesOnlyContext() throws IOException { PageCache pageCache = pageCacheRule.getPageCache( fileSystem ); - TransactionLogFilesContext context = activeFilesBuilder( directory, fileSystem, pageCache ).buildContext(); + TransactionLogFilesContext context = activeFilesBuilder( storeDirectory, fileSystem, pageCache ).buildContext(); assertEquals( fileSystem, context.getFileSystem() ); assertNotNull( context.getLogEntryReader() ); @@ -79,7 +81,7 @@ public void buildActiveFilesOnlyContext() throws IOException @Test public void buildFilesBasedContext() throws IOException { - TransactionLogFilesContext context = logFilesBasedOnlyBuilder( directory, fileSystem ).buildContext(); + TransactionLogFilesContext context = logFilesBasedOnlyBuilder( storeDirectory, fileSystem ).buildContext(); assertEquals( fileSystem, context.getFileSystem() ); assertSame( LogFileCreationMonitor.NO_MONITOR, context.getLogFileCreationMonitor() ); } @@ -87,8 +89,8 @@ public void buildFilesBasedContext() throws IOException @Test public void buildDefaultContext() throws IOException { - TransactionLogFilesContext context = - builder( directory, fileSystem ).withLogVersionRepository( new SimpleLogVersionRepository( 2 ) ) + TransactionLogFilesContext context = builder( storeDirectory, fileSystem ) + .withLogVersionRepository( new SimpleLogVersionRepository( 2 ) ) .withTransactionIdStore( new SimpleTransactionIdStore() ).buildContext(); assertEquals( fileSystem, context.getFileSystem() ); assertNotNull( context.getLogEntryReader() ); @@ -108,7 +110,7 @@ public void buildDefaultContextWithDependencies() throws IOException dependencies.satisfyDependency( transactionIdStore ); TransactionLogFilesContext context = - builder( directory, fileSystem ).withDependencies( dependencies ).buildContext(); + builder( storeDirectory, fileSystem ).withDependencies( dependencies ).buildContext(); assertEquals( fileSystem, context.getFileSystem() ); assertNotNull( context.getLogEntryReader() ); @@ -118,27 +120,55 @@ public void buildDefaultContextWithDependencies() throws IOException assertEquals( 2, context.getLogVersionRepository().getCurrentLogVersion() ); } + @Test + public void buildContextWithCustomLogFilesLocations() throws Throwable + { + String customLogLocation = "customLogLocation"; + Config customLogLocationConfig = Config.defaults( logical_logs_location, customLogLocation ); + LogFiles logFiles = builder( storeDirectory, fileSystem ).withConfig( customLogLocationConfig ) + .withLogVersionRepository( new SimpleLogVersionRepository() ) + .withTransactionIdStore( new SimpleTransactionIdStore() ).build(); + logFiles.init(); + logFiles.start(); + + assertEquals( new File( storeDirectory, customLogLocation ), logFiles.getHighestLogFile().getParentFile() ); + } + + @Test + public void buildContextWithCustomAbsoluteLogFilesLocations() throws Throwable + { + File customLogDirectory = testDirectory.directory( "absoluteCustomLogDirectory" ); + Config customLogLocationConfig = Config.defaults( logical_logs_location, customLogDirectory.getAbsolutePath() ); + LogFiles logFiles = builder( storeDirectory, fileSystem ).withConfig( customLogLocationConfig ) + .withLogVersionRepository( new SimpleLogVersionRepository() ) + .withTransactionIdStore( new SimpleTransactionIdStore() ).build(); + logFiles.init(); + logFiles.start(); + + assertEquals( customLogDirectory, logFiles.getHighestLogFile().getParentFile() ); + } + @Test( expected = NullPointerException.class ) public void failToBuildFullContextWithoutLogVersionRepo() throws IOException { - builder( directory, fileSystem ).withTransactionIdStore( new SimpleTransactionIdStore() ).buildContext(); + builder( storeDirectory, fileSystem ).withTransactionIdStore( new SimpleTransactionIdStore() ).buildContext(); } @Test( expected = NullPointerException.class ) public void failToBuildFullContextWithoutTransactionIdStore() throws IOException { - builder( directory, fileSystem ).withLogVersionRepository( new SimpleLogVersionRepository( 2 ) ).buildContext(); + builder( storeDirectory, fileSystem ).withLogVersionRepository( new SimpleLogVersionRepository( 2 ) ).buildContext(); } @Test( expected = UnsupportedOperationException.class ) public void fileBasedOperationsContextFailOnLastCommittedTransactionIdAccess() throws IOException { - logFilesBasedOnlyBuilder( directory, fileSystem ).buildContext().getLastCommittedTransactionId(); + logFilesBasedOnlyBuilder( storeDirectory, fileSystem ).buildContext().getLastCommittedTransactionId(); } @Test( expected = UnsupportedOperationException.class ) public void fileBasedOperationsContextFailOnLogVersionRepositoryAccess() throws IOException { - logFilesBasedOnlyBuilder( directory, fileSystem ).buildContext().getLogVersionRepository(); + logFilesBasedOnlyBuilder( storeDirectory, fileSystem ).buildContext().getLogVersionRepository(); } } diff --git a/community/kernel/src/test/java/org/neo4j/kernel/impl/transaction/log/files/TransactionLogFilesTest.java b/community/kernel/src/test/java/org/neo4j/kernel/impl/transaction/log/files/TransactionLogFilesTest.java index bbc1c5d426ba9..de419f073ace3 100644 --- a/community/kernel/src/test/java/org/neo4j/kernel/impl/transaction/log/files/TransactionLogFilesTest.java +++ b/community/kernel/src/test/java/org/neo4j/kernel/impl/transaction/log/files/TransactionLogFilesTest.java @@ -36,7 +36,9 @@ import static org.hamcrest.Matchers.containsInAnyOrder; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; public class TransactionLogFilesTest @@ -45,13 +47,13 @@ public class TransactionLogFilesTest public final TestDirectory testDirectory = TestDirectory.testDirectory(); @Rule public final FileSystemRule fileSystemRule = new DefaultFileSystemRule(); - private File logDirectory; + private File storeDirectory; private final String filename = "filename"; @Before public void setUp() throws Exception { - logDirectory = testDirectory.directory(); + storeDirectory = testDirectory.directory(); } @Test @@ -65,7 +67,7 @@ public void shouldGetTheFileNameForAGivenVersion() throws IOException final File versionFileName = files.getLogFileForVersion( version ); // then - final File expected = new File( logDirectory, getVersionedLogFileName( version ) ); + final File expected = new File( storeDirectory, getVersionedLogFileName( version ) ); assertEquals( expected, versionFileName ); } @@ -75,10 +77,10 @@ public void shouldVisitEachLofFile() throws IOException // given LogFiles files = createLogFiles(); - fileSystemRule.create( new File( logDirectory, getVersionedLogFileName( "1" ) ) ); - fileSystemRule.create( new File( logDirectory, getVersionedLogFileName( "some", "2" ) ) ); - fileSystemRule.create( new File( logDirectory, getVersionedLogFileName( "3" ) ) ); - fileSystemRule.create( new File( logDirectory, filename ) ); + fileSystemRule.create( new File( storeDirectory, getVersionedLogFileName( "1" ) ) ); + fileSystemRule.create( new File( storeDirectory, getVersionedLogFileName( "some", "2" ) ) ); + fileSystemRule.create( new File( storeDirectory, getVersionedLogFileName( "3" ) ) ); + fileSystemRule.create( new File( storeDirectory, filename ) ); // when final List seenFiles = new ArrayList<>(); @@ -92,8 +94,8 @@ public void shouldVisitEachLofFile() throws IOException // then assertThat( seenFiles, containsInAnyOrder( - new File( logDirectory, getVersionedLogFileName( filename, "1" ) ), - new File( logDirectory, getVersionedLogFileName( filename, "3" ) ) ) ); + new File( storeDirectory, getVersionedLogFileName( filename, "1" ) ), + new File( storeDirectory, getVersionedLogFileName( filename, "3" ) ) ) ); assertThat( seenVersions, containsInAnyOrder( 1L, 3L ) ); } @@ -103,10 +105,10 @@ public void shouldBeAbleToRetrieveTheHighestLogVersion() throws IOException // given LogFiles files = createLogFiles(); - fileSystemRule.create( new File( logDirectory, getVersionedLogFileName( "1" ) ) ); - fileSystemRule.create( new File( logDirectory, getVersionedLogFileName( "some", "4" ) ) ); - fileSystemRule.create( new File( logDirectory, getVersionedLogFileName( "3" ) ) ); - fileSystemRule.create( new File( logDirectory, filename ) ); + fileSystemRule.create( new File( storeDirectory, getVersionedLogFileName( "1" ) ) ); + fileSystemRule.create( new File( storeDirectory, getVersionedLogFileName( "some", "4" ) ) ); + fileSystemRule.create( new File( storeDirectory, getVersionedLogFileName( "3" ) ) ); + fileSystemRule.create( new File( storeDirectory, filename ) ); // when final long highestLogVersion = files.getHighestLogVersion(); @@ -121,8 +123,8 @@ public void shouldReturnANegativeValueIfThereAreNoLogFiles() throws IOException // given LogFiles files = createLogFiles(); - fileSystemRule.create( new File( logDirectory, getVersionedLogFileName( "some", "4" ) ) ); - fileSystemRule.create( new File( logDirectory, filename ) ); + fileSystemRule.create( new File( storeDirectory, getVersionedLogFileName( "some", "4" ) ) ); + fileSystemRule.create( new File( storeDirectory, filename ) ); // when final long highestLogVersion = files.getHighestLogVersion(); @@ -174,10 +176,19 @@ public void shouldThrowIfVersionIsNotANumber() throws IOException logFiles.getLogVersion( file ); } + @Test + public void isLogFile() throws IOException + { + LogFiles logFiles = createLogFiles(); + assertFalse( logFiles.isLogFile( new File( "aaa.tx.log" ) ) ); + assertTrue( logFiles.isLogFile( new File( "filename.0" ) ) ); + assertTrue( logFiles.isLogFile( new File( "filename.17" ) ) ); + } + private LogFiles createLogFiles() throws IOException { return LogFilesBuilder - .builder( logDirectory, fileSystemRule ) + .builder( storeDirectory, fileSystemRule ) .withLogFileName( filename ) .withTransactionIdStore( new SimpleTransactionIdStore() ) .withLogVersionRepository( new SimpleLogVersionRepository() ) diff --git a/community/kernel/src/test/java/org/neo4j/kernel/impl/transaction/log/stresstest/workload/Runner.java b/community/kernel/src/test/java/org/neo4j/kernel/impl/transaction/log/stresstest/workload/Runner.java index 3f3c8f5fbb041..2205581c435b8 100644 --- a/community/kernel/src/test/java/org/neo4j/kernel/impl/transaction/log/stresstest/workload/Runner.java +++ b/community/kernel/src/test/java/org/neo4j/kernel/impl/transaction/log/stresstest/workload/Runner.java @@ -70,7 +70,7 @@ public Long call() throws Exception { TransactionIdStore transactionIdStore = new SimpleTransactionIdStore(); TransactionMetadataCache transactionMetadataCache = new TransactionMetadataCache( 100_000 ); - LogFiles logFiles = life.add( createPhysicalLogFiles( transactionIdStore, fileSystem ) ); + LogFiles logFiles = life.add( createLogFiles( transactionIdStore, fileSystem ) ); TransactionAppender transactionAppender = life.add( createBatchingTransactionAppender( transactionIdStore, transactionMetadataCache, logFiles ) ); @@ -115,7 +115,7 @@ private BatchingTransactionAppender createBatchingTransactionAppender( Transacti transactionMetadataCache, transactionIdStore, IdOrderingQueue.BYPASS, databaseHealth ); } - private LogFiles createPhysicalLogFiles( TransactionIdStore transactionIdStore, + private LogFiles createLogFiles( TransactionIdStore transactionIdStore, FileSystemAbstraction fileSystemAbstraction ) throws IOException { SimpleLogVersionRepository logVersionRepository = new SimpleLogVersionRepository(); diff --git a/community/kernel/src/test/java/org/neo4j/kernel/impl/transaction/state/NeoStoreFileListingTest.java b/community/kernel/src/test/java/org/neo4j/kernel/impl/transaction/state/NeoStoreFileListingTest.java index 055526c4b9cee..baf9b45471cc6 100644 --- a/community/kernel/src/test/java/org/neo4j/kernel/impl/transaction/state/NeoStoreFileListingTest.java +++ b/community/kernel/src/test/java/org/neo4j/kernel/impl/transaction/state/NeoStoreFileListingTest.java @@ -25,6 +25,7 @@ import java.io.File; import java.io.IOException; +import java.nio.file.Paths; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -34,15 +35,20 @@ import java.util.stream.Collectors; import org.neo4j.graphdb.ResourceIterator; +import org.neo4j.graphdb.factory.GraphDatabaseSettings; import org.neo4j.kernel.NeoStoreDataSource; import org.neo4j.kernel.api.labelscan.LabelScanStore; import org.neo4j.kernel.impl.api.ExplicitIndexProviderLookup; import org.neo4j.kernel.impl.api.index.IndexingService; import org.neo4j.kernel.impl.store.StoreType; +import org.neo4j.kernel.impl.transaction.log.files.LogFiles; import org.neo4j.kernel.impl.transaction.log.files.TransactionLogFiles; +import org.neo4j.kernel.internal.GraphDatabaseAPI; import org.neo4j.storageengine.api.StorageEngine; import org.neo4j.storageengine.api.StoreFileMetadata; +import org.neo4j.test.TestGraphDatabaseFactory; import org.neo4j.test.rule.EmbeddedDatabaseRule; +import org.neo4j.test.rule.TestDirectory; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; @@ -55,7 +61,10 @@ public class NeoStoreFileListingTest { @Rule - public EmbeddedDatabaseRule db = new EmbeddedDatabaseRule(); + public final EmbeddedDatabaseRule db = new EmbeddedDatabaseRule(); + @Rule + public final TestDirectory testDirectory = TestDirectory.testDirectory(); + private NeoStoreDataSource neoStoreDataSource; private static final String[] STANDARD_STORE_DIR_FILES = new String[]{ "index", @@ -110,16 +119,6 @@ public void setUp() throws IOException neoStoreDataSource = db.getDependencyResolver().resolveDependency( NeoStoreDataSource.class ); } - private void createIndexDbFile() throws IOException - { - File storeDir = db.getStoreDir(); - final File indexFile = new File( storeDir, "index.db" ); - if ( !indexFile.exists() ) - { - assertTrue( indexFile.createNewFile() ); - } - } - @Test public void shouldCloseIndexAndLabelScanSnapshots() throws Exception { @@ -129,10 +128,11 @@ public void shouldCloseIndexAndLabelScanSnapshots() throws Exception ExplicitIndexProviderLookup explicitIndexes = mock( ExplicitIndexProviderLookup.class ); when( explicitIndexes.all() ).thenReturn( Collections.emptyList() ); File storeDir = mock( File.class ); + LogFiles logFiles = mock( LogFiles.class ); filesInStoreDirAre( storeDir, STANDARD_STORE_DIR_FILES, STANDARD_STORE_DIR_DIRECTORIES ); StorageEngine storageEngine = mock( StorageEngine.class ); - NeoStoreFileListing fileListing = new NeoStoreFileListing( - storeDir, labelScanStore, indexingService, explicitIndexes, storageEngine ); + NeoStoreFileListing fileListing = new NeoStoreFileListing( storeDir, logFiles, labelScanStore, + indexingService, explicitIndexes, storageEngine ); ResourceIterator scanSnapshot = scanStoreFilesAre( labelScanStore, new String[]{"blah/scan.store", "scan.more"} ); @@ -168,6 +168,20 @@ public void shouldListMetaDataStoreLastWithTxLogs() throws Exception .isPresent() ); } + @Test + public void shouldListTransactionLogsFromCustomLocationWhenConfigured() throws IOException + { + String logFilesPath = "customTxFolder"; + verifyLogFilesWithCustomPathListing( logFilesPath ); + } + + @Test + public void shouldListTransactionLogsFromCustomAbsoluteLocationWhenConfigured() throws IOException + { + File customLogLocation = testDirectory.directory( "customLogLocation" ); + verifyLogFilesWithCustomPathListing( customLogLocation.getAbsolutePath() ); + } + @Test public void shouldListTxLogFiles() throws Exception { @@ -198,6 +212,20 @@ public void shouldListNeostoreFiles() throws Exception assertEquals( Arrays.asList(values), listedStoreFiles ); } + private void verifyLogFilesWithCustomPathListing( String path ) throws IOException + { + GraphDatabaseAPI graphDatabase = (GraphDatabaseAPI) new TestGraphDatabaseFactory() + .newEmbeddedDatabaseBuilder( testDirectory.directory( "customDb" ) ) + .setConfig( GraphDatabaseSettings.logical_logs_location, path ) + .newGraphDatabase(); + NeoStoreDataSource dataSource = graphDatabase.getDependencyResolver().resolveDependency( NeoStoreDataSource.class ); + LogFiles logFiles = graphDatabase.getDependencyResolver().resolveDependency( LogFiles.class ); + assertTrue( dataSource.listStoreFiles( true ).stream() + .anyMatch( metadata -> metadata.isLogFile() && logFiles.isLogFile( metadata.file() ) ) ); + assertEquals( Paths.get( path ).getFileName().toString(), logFiles.logFilesDirectory().getName() ); + graphDatabase.shutdown(); + } + private void filesInStoreDirAre( File storeDir, String[] filenames, String[] dirs ) { ArrayList files = new ArrayList<>(); @@ -226,6 +254,16 @@ private ResourceIterator indexFilesAre( IndexingService indexingService, S return snapshot; } + private void createIndexDbFile() throws IOException + { + File storeDir = db.getStoreDir(); + final File indexFile = new File( storeDir, "index.db" ); + if ( !indexFile.exists() ) + { + assertTrue( indexFile.createNewFile() ); + } + } + private void mockFiles( String[] filenames, ArrayList files, boolean isDirectories ) { for ( String filename : filenames ) diff --git a/community/kernel/src/test/java/org/neo4j/test/AdversarialPageCacheGraphDatabaseFactory.java b/community/kernel/src/test/java/org/neo4j/test/AdversarialPageCacheGraphDatabaseFactory.java index 1ec92f82405d3..f00c9dfebbbe0 100644 --- a/community/kernel/src/test/java/org/neo4j/test/AdversarialPageCacheGraphDatabaseFactory.java +++ b/community/kernel/src/test/java/org/neo4j/test/AdversarialPageCacheGraphDatabaseFactory.java @@ -20,12 +20,12 @@ package org.neo4j.test; import java.io.File; -import java.util.Map; import org.neo4j.adversaries.Adversary; import org.neo4j.adversaries.pagecache.AdversarialPageCache; import org.neo4j.graphdb.GraphDatabaseService; import org.neo4j.graphdb.factory.GraphDatabaseFactory; +import org.neo4j.graphdb.factory.GraphDatabaseSettings; import org.neo4j.io.fs.FileSystemAbstraction; import org.neo4j.io.pagecache.PageCache; import org.neo4j.kernel.configuration.Config; @@ -61,6 +61,7 @@ protected GraphDatabaseService newEmbeddedDatabase( File dir, Config config, Dep protected PlatformModule createPlatform( File storeDir, Config config, Dependencies dependencies, GraphDatabaseFacade facade ) { + config.augment( GraphDatabaseSettings.database_path, storeDir.getAbsolutePath() ); return new PlatformModule( storeDir, config, databaseInfo, dependencies, facade ) { @Override diff --git a/community/kernel/src/test/java/org/neo4j/test/TestGraphDatabaseFactory.java b/community/kernel/src/test/java/org/neo4j/test/TestGraphDatabaseFactory.java index 96143e53ce854..6b2afa348d5ed 100644 --- a/community/kernel/src/test/java/org/neo4j/test/TestGraphDatabaseFactory.java +++ b/community/kernel/src/test/java/org/neo4j/test/TestGraphDatabaseFactory.java @@ -259,6 +259,7 @@ protected TestGraphDatabaseFacadeFactory( TestGraphDatabaseFactoryState state, b protected PlatformModule createPlatform( File storeDir, Config config, Dependencies dependencies, GraphDatabaseFacade graphDatabaseFacade ) { + config.augment( GraphDatabaseSettings.database_path, storeDir.getAbsolutePath() ); if ( impermanent ) { config.augment( ephemeral, TRUE ); diff --git a/community/neo4j-harness/src/main/java/org/neo4j/harness/internal/AbstractInProcessServerBuilder.java b/community/neo4j-harness/src/main/java/org/neo4j/harness/internal/AbstractInProcessServerBuilder.java index 27d7d6b85a3d5..513fc6940fd97 100644 --- a/community/neo4j-harness/src/main/java/org/neo4j/harness/internal/AbstractInProcessServerBuilder.java +++ b/community/neo4j-harness/src/main/java/org/neo4j/harness/internal/AbstractInProcessServerBuilder.java @@ -54,8 +54,8 @@ import org.neo4j.server.configuration.ServerSettings; import org.neo4j.server.configuration.ThirdPartyJaxRsPackage; -import static org.neo4j.dbms.DatabaseManagementSystemSettings.data_directory; import static org.neo4j.graphdb.factory.GraphDatabaseSettings.auth_enabled; +import static org.neo4j.graphdb.factory.GraphDatabaseSettings.data_directory; import static org.neo4j.graphdb.factory.GraphDatabaseSettings.pagecache_memory; import static org.neo4j.helpers.collection.Iterables.append; import static org.neo4j.io.file.Files.createOrOpenAsOuputStream; diff --git a/community/neo4j-harness/src/test/java/org/neo4j/harness/InProcessBuilderTestIT.java b/community/neo4j-harness/src/test/java/org/neo4j/harness/InProcessBuilderTestIT.java index d0bf0c6bbfd35..15ec2c30f3ab5 100644 --- a/community/neo4j-harness/src/test/java/org/neo4j/harness/InProcessBuilderTestIT.java +++ b/community/neo4j-harness/src/test/java/org/neo4j/harness/InProcessBuilderTestIT.java @@ -42,7 +42,6 @@ import javax.net.ssl.X509TrustManager; import org.neo4j.bolt.v1.transport.socket.client.SocketConnection; -import org.neo4j.dbms.DatabaseManagementSystemSettings; import org.neo4j.graphdb.GraphDatabaseService; import org.neo4j.graphdb.Node; import org.neo4j.graphdb.ResourceIterable; @@ -179,8 +178,8 @@ public void shouldRunBuilderOnExistingStoreDir() throws Exception // When // create graph db with one node upfront Path dir = Files.createTempDirectory( getClass().getSimpleName() + "_shouldRunBuilderOnExistingStorageDir" ); - File storeDir = Config.defaults( DatabaseManagementSystemSettings.data_directory, dir.toString() ) - .get( DatabaseManagementSystemSettings.database_path ); + File storeDir = Config.defaults( GraphDatabaseSettings.data_directory, dir.toString() ) + .get( GraphDatabaseSettings.database_path ); try { GraphDatabaseService db = new TestGraphDatabaseFactory().newEmbeddedDatabase( storeDir ); diff --git a/community/server/src/main/java/org/neo4j/server/CommunityNeoServer.java b/community/server/src/main/java/org/neo4j/server/CommunityNeoServer.java index 3ed4a087a0993..8d6179e39cd19 100644 --- a/community/server/src/main/java/org/neo4j/server/CommunityNeoServer.java +++ b/community/server/src/main/java/org/neo4j/server/CommunityNeoServer.java @@ -24,7 +24,7 @@ import java.util.Arrays; import java.util.List; -import org.neo4j.dbms.DatabaseManagementSystemSettings; +import org.neo4j.graphdb.factory.GraphDatabaseSettings; import org.neo4j.kernel.configuration.Config; import org.neo4j.kernel.impl.factory.CommunityEditionModule; import org.neo4j.kernel.impl.factory.DatabaseInfo; @@ -53,7 +53,7 @@ public class CommunityNeoServer extends AbstractNeoServer { protected static final GraphFactory COMMUNITY_FACTORY = ( config, dependencies ) -> { - File storeDir = config.get( DatabaseManagementSystemSettings.database_path ); + File storeDir = config.get( GraphDatabaseSettings.database_path ); return new GraphDatabaseFacadeFactory( DatabaseInfo.COMMUNITY, CommunityEditionModule::new ) .newFacade( storeDir, config, dependencies ); }; diff --git a/community/server/src/main/java/org/neo4j/server/database/LifecycleManagingDatabase.java b/community/server/src/main/java/org/neo4j/server/database/LifecycleManagingDatabase.java index 712ace68d13b9..d7596b350559c 100644 --- a/community/server/src/main/java/org/neo4j/server/database/LifecycleManagingDatabase.java +++ b/community/server/src/main/java/org/neo4j/server/database/LifecycleManagingDatabase.java @@ -21,8 +21,8 @@ import java.io.File; -import org.neo4j.dbms.DatabaseManagementSystemSettings; import org.neo4j.graphdb.Result; +import org.neo4j.graphdb.factory.GraphDatabaseSettings; import org.neo4j.kernel.configuration.Config; import org.neo4j.kernel.impl.factory.GraphDatabaseFacade; import org.neo4j.kernel.impl.factory.GraphDatabaseFacadeFactory; @@ -67,7 +67,7 @@ public LifecycleManagingDatabase( Config config, GraphFactory dbFactory, @Override public File getLocation() { - return config.get( DatabaseManagementSystemSettings.database_path ); + return config.get( GraphDatabaseSettings.database_path ); } @Override diff --git a/community/server/src/test/java/org/neo4j/server/BaseBootstrapperTestIT.java b/community/server/src/test/java/org/neo4j/server/BaseBootstrapperTestIT.java index 526738b26fbc0..a601a4ff9eb8d 100644 --- a/community/server/src/test/java/org/neo4j/server/BaseBootstrapperTestIT.java +++ b/community/server/src/test/java/org/neo4j/server/BaseBootstrapperTestIT.java @@ -41,7 +41,7 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertThat; import static org.neo4j.bolt.v1.transport.integration.Neo4jWithSocket.DEFAULT_CONNECTOR_KEY; -import static org.neo4j.dbms.DatabaseManagementSystemSettings.data_directory; +import static org.neo4j.graphdb.factory.GraphDatabaseSettings.data_directory; import static org.neo4j.graphdb.factory.GraphDatabaseSettings.forced_kernel_id; import static org.neo4j.graphdb.factory.GraphDatabaseSettings.logs_directory; import static org.neo4j.helpers.collection.MapUtil.store; diff --git a/community/server/src/test/java/org/neo4j/server/ServerBootstrapperTest.java b/community/server/src/test/java/org/neo4j/server/ServerBootstrapperTest.java index 6b6cb2f7f0c61..0dee20f456f93 100644 --- a/community/server/src/test/java/org/neo4j/server/ServerBootstrapperTest.java +++ b/community/server/src/test/java/org/neo4j/server/ServerBootstrapperTest.java @@ -27,7 +27,6 @@ import java.util.Collection; import java.util.Collections; import java.util.Optional; - import javax.annotation.Nonnull; import org.neo4j.helpers.collection.MapUtil; @@ -42,8 +41,7 @@ import static org.hamcrest.Matchers.not; import static org.junit.Assert.assertThat; import static org.mockito.Mockito.mock; - -import static org.neo4j.dbms.DatabaseManagementSystemSettings.database_path; +import static org.neo4j.graphdb.factory.GraphDatabaseSettings.database_path; public class ServerBootstrapperTest { diff --git a/community/server/src/test/java/org/neo4j/server/ServerTestUtils.java b/community/server/src/test/java/org/neo4j/server/ServerTestUtils.java index 48047e9d98b85..569a8d95ebfe8 100644 --- a/community/server/src/test/java/org/neo4j/server/ServerTestUtils.java +++ b/community/server/src/test/java/org/neo4j/server/ServerTestUtils.java @@ -33,7 +33,6 @@ import java.util.Properties; import java.util.Random; -import org.neo4j.dbms.DatabaseManagementSystemSettings; import org.neo4j.graphdb.config.Setting; import org.neo4j.graphdb.factory.GraphDatabaseSettings; import org.neo4j.kernel.configuration.ssl.LegacySslPolicyConfig; @@ -90,7 +89,7 @@ public static Map getDefaultRelativeProperties() throws IOExcepti public static void addDefaultRelativeProperties( Map properties, File temporaryFolder ) { - addRelativeProperty( temporaryFolder, properties, DatabaseManagementSystemSettings.data_directory ); + addRelativeProperty( temporaryFolder, properties, GraphDatabaseSettings.data_directory ); addRelativeProperty( temporaryFolder, properties, GraphDatabaseSettings.logs_directory ); addRelativeProperty( temporaryFolder, properties, LegacySslPolicyConfig.certificates_directory ); properties.put( GraphDatabaseSettings.pagecache_memory.name(), "8m" ); diff --git a/community/server/src/test/java/org/neo4j/server/configuration/ConfigFileBuilder.java b/community/server/src/test/java/org/neo4j/server/configuration/ConfigFileBuilder.java index 412781451f01b..9c870b94b8bc8 100644 --- a/community/server/src/test/java/org/neo4j/server/configuration/ConfigFileBuilder.java +++ b/community/server/src/test/java/org/neo4j/server/configuration/ConfigFileBuilder.java @@ -23,8 +23,8 @@ import java.io.IOException; import java.util.Map; -import org.neo4j.dbms.DatabaseManagementSystemSettings; import org.neo4j.graphdb.config.Setting; +import org.neo4j.graphdb.factory.GraphDatabaseSettings; import org.neo4j.helpers.collection.MapUtil; import org.neo4j.server.ServerTestUtils; @@ -45,7 +45,7 @@ private ConfigFileBuilder( File directory ) //initialize config with defaults that doesn't pollute //workspace with generated data this.config = MapUtil.stringMap( - DatabaseManagementSystemSettings.data_directory.name(), directory.getAbsolutePath() + "/data", + GraphDatabaseSettings.data_directory.name(), directory.getAbsolutePath() + "/data", ServerSettings.management_api_path.name(), "http://localhost:7474/db/manage/", ServerSettings.rest_api_path.name(), "http://localhost:7474/db/data/" ); } diff --git a/community/server/src/test/java/org/neo4j/server/configuration/ConfigLoaderTest.java b/community/server/src/test/java/org/neo4j/server/configuration/ConfigLoaderTest.java index 1698b54fe3f91..9438319f35446 100644 --- a/community/server/src/test/java/org/neo4j/server/configuration/ConfigLoaderTest.java +++ b/community/server/src/test/java/org/neo4j/server/configuration/ConfigLoaderTest.java @@ -222,7 +222,7 @@ public void shouldDefaultToCorrectValueForAuthStoreLocation() throws IOException { File configFile = ConfigFileBuilder .builder( folder.getRoot() ) - .withoutSetting( DatabaseManagementSystemSettings.data_directory ) + .withoutSetting( GraphDatabaseSettings.data_directory ) .build(); Config config = Config.fromFile( configFile ).withHome( folder.getRoot() ).build(); @@ -234,7 +234,7 @@ public void shouldDefaultToCorrectValueForAuthStoreLocation() throws IOException public void shouldSetAValueForAuthStoreLocation() throws IOException { File configFile = ConfigFileBuilder.builder( folder.getRoot() ) - .withSetting( DatabaseManagementSystemSettings.data_directory, "the-data-dir" ) + .withSetting( GraphDatabaseSettings.data_directory, "the-data-dir" ) .build(); Config config = Config.fromFile( configFile ).withHome( folder.getRoot() ).build(); @@ -246,7 +246,7 @@ public void shouldSetAValueForAuthStoreLocation() throws IOException public void shouldNotOverwriteAuthStoreLocationIfProvided() throws IOException { File configFile = ConfigFileBuilder.builder( folder.getRoot() ) - .withSetting( DatabaseManagementSystemSettings.data_directory, "the-data-dir" ) + .withSetting( GraphDatabaseSettings.data_directory, "the-data-dir" ) .withSetting( GraphDatabaseSettings.auth_store, "foo/bar/auth" ) .build(); Config config = Config.fromFile( configFile ).withHome( folder.getRoot() ).build(); diff --git a/community/server/src/test/java/org/neo4j/server/database/TestLifecycleManagedDatabase.java b/community/server/src/test/java/org/neo4j/server/database/TestLifecycleManagedDatabase.java index f1bcf7f97dc57..255499b4b75bc 100644 --- a/community/server/src/test/java/org/neo4j/server/database/TestLifecycleManagedDatabase.java +++ b/community/server/src/test/java/org/neo4j/server/database/TestLifecycleManagedDatabase.java @@ -28,7 +28,7 @@ import java.io.File; import java.io.IOException; -import org.neo4j.dbms.DatabaseManagementSystemSettings; +import org.neo4j.graphdb.factory.GraphDatabaseSettings; import org.neo4j.kernel.GraphDatabaseDependencies; import org.neo4j.kernel.StoreLockException; import org.neo4j.kernel.configuration.Config; @@ -66,7 +66,7 @@ public void setup() throws Exception dataDirectory = createTempDir(); dbFactory = createGraphFactory(); - dbConfig = Config.defaults( DatabaseManagementSystemSettings.data_directory, dataDirectory.getAbsolutePath() ); + dbConfig = Config.defaults( GraphDatabaseSettings.data_directory, dataDirectory.getAbsolutePath() ); theDatabase = newDatabase(); } @@ -142,7 +142,7 @@ public void shouldBeAbleToGetLocation() throws Throwable { theDatabase.start(); assertThat( theDatabase.getLocation().getAbsolutePath(), - is( dbConfig.get( DatabaseManagementSystemSettings.database_path ).getAbsolutePath() ) ); + is( dbConfig.get( GraphDatabaseSettings.database_path ).getAbsolutePath() ) ); } private LifecycleManagingDatabase.GraphFactory createGraphFactory() diff --git a/community/server/src/test/java/org/neo4j/server/helpers/CommunityServerBuilder.java b/community/server/src/test/java/org/neo4j/server/helpers/CommunityServerBuilder.java index f9c4debc4e91f..2f3cb007f8265 100644 --- a/community/server/src/test/java/org/neo4j/server/helpers/CommunityServerBuilder.java +++ b/community/server/src/test/java/org/neo4j/server/helpers/CommunityServerBuilder.java @@ -28,7 +28,6 @@ import java.util.Map; import java.util.Properties; -import org.neo4j.dbms.DatabaseManagementSystemSettings; import org.neo4j.graphdb.factory.GraphDatabaseSettings; import org.neo4j.helpers.ListenSocketAddress; import org.neo4j.kernel.GraphDatabaseDependencies; @@ -79,7 +78,7 @@ public class CommunityServerBuilder private static LifecycleManagingDatabase.GraphFactory IN_MEMORY_DB = ( config, dependencies ) -> { - File storeDir = config.get( DatabaseManagementSystemSettings.database_path ); + File storeDir = config.get( GraphDatabaseSettings.database_path ); config.augment( stringMap( GraphDatabaseFacadeFactory.Configuration.ephemeral.name(), "true", new BoltConnector( "bolt" ).listen_address.name(), "localhost:0" ) ); return new ImpermanentGraphDatabase( storeDir, config, @@ -155,7 +154,7 @@ public Map createConfiguration( File temporaryFolder ) if ( dataDir != null ) { - properties.put( DatabaseManagementSystemSettings.data_directory.name(), dataDir ); + properties.put( GraphDatabaseSettings.data_directory.name(), dataDir ); } if ( maxThreads != null ) @@ -206,6 +205,8 @@ public Map createConfiguration( File temporaryFolder ) new File( temporaryFolder, "certificates" ).getAbsolutePath() ); properties.put( GraphDatabaseSettings.logs_directory.name(), new File( temporaryFolder, "logs" ).getAbsolutePath() ); + properties.put( GraphDatabaseSettings.logical_logs_location.name(), + new File( temporaryFolder, "transaction-logs" ).getAbsolutePath() ); properties.put( GraphDatabaseSettings.pagecache_memory.name(), "8m" ); for ( Object key : arbitraryProperties.keySet() ) diff --git a/community/server/src/test/java/org/neo4j/server/integration/StartupLoggingIT.java b/community/server/src/test/java/org/neo4j/server/integration/StartupLoggingIT.java index 62a8ea318698e..357bf43a61a0b 100644 --- a/community/server/src/test/java/org/neo4j/server/integration/StartupLoggingIT.java +++ b/community/server/src/test/java/org/neo4j/server/integration/StartupLoggingIT.java @@ -33,7 +33,6 @@ import java.util.Map; import java.util.Optional; -import org.neo4j.dbms.DatabaseManagementSystemSettings; import org.neo4j.graphdb.factory.GraphDatabaseSettings; import org.neo4j.io.fs.FileUtils; import org.neo4j.kernel.configuration.BoltConnector; @@ -47,10 +46,8 @@ import org.neo4j.test.rule.TestDirectory; import org.neo4j.test.server.ExclusiveServerTestBase; -import static org.hamcrest.MatcherAssert.assertThat; - import static java.util.Arrays.asList; - +import static org.hamcrest.MatcherAssert.assertThat; import static org.neo4j.bolt.v1.transport.integration.Neo4jWithSocket.DEFAULT_CONNECTOR_KEY; import static org.neo4j.server.AbstractNeoServer.NEO4J_IS_STARTING_MESSAGE; @@ -62,7 +59,7 @@ public class StartupLoggingIT extends ExclusiveServerTestBase @Before public void setUp() throws IOException { - FileUtils.deleteRecursively( ServerTestUtils.getRelativeFile( DatabaseManagementSystemSettings.data_directory ) ); + FileUtils.deleteRecursively( ServerTestUtils.getRelativeFile( GraphDatabaseSettings.data_directory ) ); } @Rule @@ -111,7 +108,7 @@ private Map getPropertyPairs() throws IOException relativeProperties.put( bolt.enabled.name(), "true" ); relativeProperties.put( bolt.listen_address.name(), "localhost:" + PortAuthority.allocatePort() ); - relativeProperties.put( DatabaseManagementSystemSettings.database_path.name(), + relativeProperties.put( GraphDatabaseSettings.database_path.name(), homeDir.absolutePath().getAbsolutePath() ); return relativeProperties; } diff --git a/enterprise/backup/src/main/java/org/neo4j/backup/AbstractBackupSupportingClassesFactory.java b/enterprise/backup/src/main/java/org/neo4j/backup/AbstractBackupSupportingClassesFactory.java index 431ae532aea5c..488b6b9dee6c8 100644 --- a/enterprise/backup/src/main/java/org/neo4j/backup/AbstractBackupSupportingClassesFactory.java +++ b/enterprise/backup/src/main/java/org/neo4j/backup/AbstractBackupSupportingClassesFactory.java @@ -29,13 +29,10 @@ import org.neo4j.causalclustering.catchup.tx.TxPullClient; import org.neo4j.causalclustering.handlers.PipelineHandlerAppender; import org.neo4j.causalclustering.handlers.PipelineHandlerAppenderFactory; -import org.neo4j.graphdb.DependencyResolver; import org.neo4j.io.fs.FileSystemAbstraction; import org.neo4j.io.pagecache.PageCache; import org.neo4j.kernel.configuration.Config; -import org.neo4j.kernel.configuration.ssl.SslPolicyLoader; import org.neo4j.kernel.impl.pagecache.ConfigurableStandalonePageCacheFactory; -import org.neo4j.kernel.impl.transaction.state.DataSourceManager; import org.neo4j.kernel.impl.util.Dependencies; import org.neo4j.kernel.monitoring.Monitors; import org.neo4j.logging.LogProvider; @@ -94,8 +91,8 @@ private BackupDelegator backupDelegatorFromConfig( PageCache pageCache, Config c TxPullClient txPullClient = new TxPullClient( catchUpClient, monitors ); StoreCopyClient storeCopyClient = new StoreCopyClient( catchUpClient, logProvider ); - RemoteStore remoteStore = - new RemoteStore( logProvider, fileSystemAbstraction, pageCache, storeCopyClient, txPullClient, transactionLogCatchUpFactory, monitors ); + RemoteStore remoteStore = new RemoteStore( logProvider, fileSystemAbstraction, pageCache, storeCopyClient, + txPullClient, transactionLogCatchUpFactory, config, monitors ); return backupDelegator( remoteStore, catchUpClient, storeCopyClient ); } diff --git a/enterprise/backup/src/main/java/org/neo4j/backup/BackupDelegator.java b/enterprise/backup/src/main/java/org/neo4j/backup/BackupDelegator.java index ecce952610af8..73a98fa0b3d9e 100644 --- a/enterprise/backup/src/main/java/org/neo4j/backup/BackupDelegator.java +++ b/enterprise/backup/src/main/java/org/neo4j/backup/BackupDelegator.java @@ -70,7 +70,7 @@ CatchupResult tryCatchingUp( AdvertisedSocketAddress fromAddress, StoreId expect { try { - return remoteStore.tryCatchingUp( fromAddress, expectedStoreId, storeDir ); + return remoteStore.tryCatchingUp( fromAddress, expectedStoreId, storeDir, true ); } catch ( IOException e ) { diff --git a/enterprise/backup/src/main/java/org/neo4j/backup/BackupProtocolService.java b/enterprise/backup/src/main/java/org/neo4j/backup/BackupProtocolService.java index 5994ba2ac66c2..efed81c97acae 100644 --- a/enterprise/backup/src/main/java/org/neo4j/backup/BackupProtocolService.java +++ b/enterprise/backup/src/main/java/org/neo4j/backup/BackupProtocolService.java @@ -27,7 +27,6 @@ import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.Optional; import java.util.function.Supplier; import javax.annotation.Nullable; @@ -80,7 +79,6 @@ import static org.neo4j.com.storecopy.TransactionCommittingResponseUnpacker.DEFAULT_BATCH_SIZE; import static org.neo4j.graphdb.factory.GraphDatabaseSettings.logs_directory; import static org.neo4j.graphdb.factory.GraphDatabaseSettings.store_internal_log_path; -import static org.neo4j.helpers.Exceptions.launderedException; import static org.neo4j.helpers.Exceptions.rootCause; import static org.neo4j.kernel.impl.pagecache.ConfigurableStandalonePageCacheFactory.createPageCache; diff --git a/enterprise/backup/src/main/java/org/neo4j/backup/BackupRecoveryService.java b/enterprise/backup/src/main/java/org/neo4j/backup/BackupRecoveryService.java index 5c20af3748f59..ffaf6379c12fd 100644 --- a/enterprise/backup/src/main/java/org/neo4j/backup/BackupRecoveryService.java +++ b/enterprise/backup/src/main/java/org/neo4j/backup/BackupRecoveryService.java @@ -22,6 +22,7 @@ import java.io.File; import java.util.Map; +import org.neo4j.graphdb.factory.GraphDatabaseSettings; import org.neo4j.io.pagecache.PageCache; import org.neo4j.kernel.configuration.Config; import org.neo4j.kernel.internal.GraphDatabaseAPI; @@ -33,6 +34,7 @@ public class BackupRecoveryService public void recoverWithDatabase( File targetDirectory, PageCache pageCache, Config config ) { Map configParams = config.getRaw(); + configParams.put( GraphDatabaseSettings.logical_logs_location.name(), targetDirectory.getAbsolutePath() ); GraphDatabaseAPI targetDb = startTemporaryDb( targetDirectory, pageCache, configParams ); targetDb.shutdown(); } diff --git a/enterprise/backup/src/main/java/org/neo4j/backup/BackupStrategyWrapper.java b/enterprise/backup/src/main/java/org/neo4j/backup/BackupStrategyWrapper.java index 3bb8c11d98813..6495b32fbf6f1 100644 --- a/enterprise/backup/src/main/java/org/neo4j/backup/BackupStrategyWrapper.java +++ b/enterprise/backup/src/main/java/org/neo4j/backup/BackupStrategyWrapper.java @@ -20,17 +20,13 @@ package org.neo4j.backup; import java.io.File; -import java.util.Map; import org.neo4j.commandline.admin.CommandFailed; import org.neo4j.helpers.OptionalHostnamePort; import org.neo4j.io.pagecache.PageCache; import org.neo4j.kernel.configuration.Config; -import org.neo4j.kernel.internal.GraphDatabaseAPI; import org.neo4j.kernel.lifecycle.LifeSupport; -import static org.neo4j.backup.BackupProtocolService.startTemporaryDb; - class BackupStrategyWrapper { private final BackupStrategy backupStrategy; diff --git a/enterprise/backup/src/main/java/org/neo4j/backup/OnlineBackupCommandConfigLoader.java b/enterprise/backup/src/main/java/org/neo4j/backup/OnlineBackupCommandConfigLoader.java index ebb6032eb6446..a1b80478f06ef 100644 --- a/enterprise/backup/src/main/java/org/neo4j/backup/OnlineBackupCommandConfigLoader.java +++ b/enterprise/backup/src/main/java/org/neo4j/backup/OnlineBackupCommandConfigLoader.java @@ -40,8 +40,9 @@ class OnlineBackupCommandConfigLoader Config loadConfig( Optional additionalConfig ) throws CommandFailed { - return withAdditionalConfig( additionalConfig, - Config.fromFile( configDir.resolve( Config.DEFAULT_CONFIG_FILE_NAME ) ).withHome( homeDir ).withConnectorsDisabled().build() ); + Config config = Config.fromFile( configDir.resolve( Config.DEFAULT_CONFIG_FILE_NAME ) ).withHome( homeDir ) + .withConnectorsDisabled().build(); + return withAdditionalConfig( additionalConfig, config ); } private Config withAdditionalConfig( Optional additionalConfig, Config config ) throws CommandFailed diff --git a/enterprise/backup/src/main/java/org/neo4j/backup/OnlineBackupRequiredArguments.java b/enterprise/backup/src/main/java/org/neo4j/backup/OnlineBackupRequiredArguments.java index 3b99ec576c95c..6c0d287fa0caa 100644 --- a/enterprise/backup/src/main/java/org/neo4j/backup/OnlineBackupRequiredArguments.java +++ b/enterprise/backup/src/main/java/org/neo4j/backup/OnlineBackupRequiredArguments.java @@ -36,8 +36,7 @@ public class OnlineBackupRequiredArguments private final Path reportDir; public OnlineBackupRequiredArguments( OptionalHostnamePort address, Path folder, String name, boolean fallbackToFull, boolean doConsistencyCheck, - long timeout, - Optional additionalConfig, Path reportDir ) + long timeout, Optional additionalConfig, Path reportDir ) { this.address = address; this.folder = folder; diff --git a/enterprise/backup/src/main/java/org/neo4j/restore/RestoreDatabaseCli.java b/enterprise/backup/src/main/java/org/neo4j/restore/RestoreDatabaseCli.java index a57bf1fff5150..9f59bcbb58fc7 100644 --- a/enterprise/backup/src/main/java/org/neo4j/restore/RestoreDatabaseCli.java +++ b/enterprise/backup/src/main/java/org/neo4j/restore/RestoreDatabaseCli.java @@ -29,7 +29,7 @@ import org.neo4j.commandline.arguments.Arguments; import org.neo4j.commandline.arguments.MandatoryNamedArg; import org.neo4j.commandline.arguments.OptionalBooleanArg; -import org.neo4j.dbms.DatabaseManagementSystemSettings; +import org.neo4j.graphdb.factory.GraphDatabaseSettings; import org.neo4j.io.fs.DefaultFileSystemAbstraction; import org.neo4j.io.fs.FileSystemAbstraction; import org.neo4j.kernel.configuration.Config; @@ -53,7 +53,7 @@ private static Config loadNeo4jConfig( Path homeDir, Path configDir, String data { return Config.fromFile( configDir.resolve( Config.DEFAULT_CONFIG_FILE_NAME ) ) .withHome( homeDir ) - .withSetting( DatabaseManagementSystemSettings.active_database, databaseName ) + .withSetting( GraphDatabaseSettings.active_database, databaseName ) .withConnectorsDisabled().build(); } diff --git a/enterprise/backup/src/main/java/org/neo4j/restore/RestoreDatabaseCommand.java b/enterprise/backup/src/main/java/org/neo4j/restore/RestoreDatabaseCommand.java index 4e0c35596bd87..50cb4808bb4e8 100644 --- a/enterprise/backup/src/main/java/org/neo4j/restore/RestoreDatabaseCommand.java +++ b/enterprise/backup/src/main/java/org/neo4j/restore/RestoreDatabaseCommand.java @@ -23,30 +23,36 @@ import java.io.IOException; import org.neo4j.commandline.admin.CommandFailed; +import org.neo4j.graphdb.factory.GraphDatabaseSettings; import org.neo4j.io.fs.FileSystemAbstraction; import org.neo4j.kernel.configuration.Config; +import org.neo4j.kernel.impl.transaction.log.files.LogFiles; +import org.neo4j.kernel.impl.transaction.log.files.LogFilesBuilder; import org.neo4j.kernel.impl.util.Validators; import static java.lang.String.format; import static org.neo4j.commandline.Util.checkLock; -import static org.neo4j.dbms.DatabaseManagementSystemSettings.database_path; +import static org.neo4j.commandline.Util.isSameOrChildFile; +import static org.neo4j.graphdb.factory.GraphDatabaseSettings.database_path; public class RestoreDatabaseCommand { private FileSystemAbstraction fs; private final File fromPath; private final File databaseDir; + private final File transactionLogsDirectory; private String databaseName; private boolean forceOverwrite; - public RestoreDatabaseCommand( FileSystemAbstraction fs, File fromPath, Config config, - String databaseName, boolean forceOverwrite ) + public RestoreDatabaseCommand( FileSystemAbstraction fs, File fromPath, Config config, String databaseName, + boolean forceOverwrite ) { this.fs = fs; this.fromPath = fromPath; this.databaseName = databaseName; this.forceOverwrite = forceOverwrite; this.databaseDir = config.get( database_path ).getAbsoluteFile(); + this.transactionLogsDirectory = config.get( GraphDatabaseSettings.logical_logs_location ).getAbsoluteFile(); } public void execute() throws IOException, CommandFailed @@ -75,6 +81,19 @@ public void execute() throws IOException, CommandFailed checkLock( databaseDir.toPath() ); fs.deleteRecursively( databaseDir ); - fs.copyRecursively( fromPath, databaseDir ); + + if ( !isSameOrChildFile( databaseDir, transactionLogsDirectory ) ) + { + fs.deleteRecursively( transactionLogsDirectory ); + } + LogFiles backupLogFiles = LogFilesBuilder.logFilesBasedOnlyBuilder( fromPath, fs ).build(); + File[] files = fromPath.listFiles(); + if ( files != null ) + { + for ( File file : files ) + { + fs.copyToDirectory( file, backupLogFiles.isLogFile( file ) ? transactionLogsDirectory : databaseDir ); + } + } } } diff --git a/enterprise/backup/src/test/java/org/neo4j/backup/BackupDelegatorTest.java b/enterprise/backup/src/test/java/org/neo4j/backup/BackupDelegatorTest.java index 54004dc8f2c5a..c5e410c77b370 100644 --- a/enterprise/backup/src/test/java/org/neo4j/backup/BackupDelegatorTest.java +++ b/enterprise/backup/src/test/java/org/neo4j/backup/BackupDelegatorTest.java @@ -73,7 +73,7 @@ public void tryCatchingUpDelegatesToRemoteStore() throws org.neo4j.causalcluster subject.tryCatchingUp( fromAddress, expectedStoreId, storeDir ); // then - verify( remoteStore ).tryCatchingUp( fromAddress, expectedStoreId, storeDir ); + verify( remoteStore ).tryCatchingUp( fromAddress, expectedStoreId, storeDir, true ); } @Test diff --git a/enterprise/backup/src/test/java/org/neo4j/backup/BackupFlowTest.java b/enterprise/backup/src/test/java/org/neo4j/backup/BackupFlowTest.java index c3565908b09b9..8a013c24ec798 100644 --- a/enterprise/backup/src/test/java/org/neo4j/backup/BackupFlowTest.java +++ b/enterprise/backup/src/test/java/org/neo4j/backup/BackupFlowTest.java @@ -35,6 +35,7 @@ import org.neo4j.consistency.checking.full.ConsistencyCheckIncompleteException; import org.neo4j.consistency.checking.full.ConsistencyFlags; import org.neo4j.helpers.progress.ProgressMonitorFactory; +import org.neo4j.io.fs.FileSystemAbstraction; import org.neo4j.logging.LogProvider; import static org.hamcrest.CoreMatchers.equalTo; @@ -53,26 +54,28 @@ public class BackupFlowTest public ExpectedException expectedException = ExpectedException.none(); // dependencies - final ConsistencyCheckService consistencyCheckService = mock( ConsistencyCheckService.class ); - final OutsideWorld outsideWorld = mock( OutsideWorld.class ); - final LogProvider logProvider = mock( LogProvider.class ); - final BackupStrategyWrapper firstStrategy = mock( BackupStrategyWrapper.class ); - final BackupStrategyWrapper secondStrategy = mock( BackupStrategyWrapper.class ); + private final ConsistencyCheckService consistencyCheckService = mock( ConsistencyCheckService.class ); + private final OutsideWorld outsideWorld = mock( OutsideWorld.class ); + private final FileSystemAbstraction fileSystem = mock( FileSystemAbstraction.class ); + private final LogProvider logProvider = mock( LogProvider.class ); + private final BackupStrategyWrapper firstStrategy = mock( BackupStrategyWrapper.class ); + private final BackupStrategyWrapper secondStrategy = mock( BackupStrategyWrapper.class ); BackupFlow subject; // test method parameter mocks - final OnlineBackupContext onlineBackupContext = mock( OnlineBackupContext.class ); - final OnlineBackupRequiredArguments requiredArguments = mock( OnlineBackupRequiredArguments.class ); + private final OnlineBackupContext onlineBackupContext = mock( OnlineBackupContext.class ); + private final OnlineBackupRequiredArguments requiredArguments = mock( OnlineBackupRequiredArguments.class ); // mock returns - private ProgressMonitorFactory progressMonitorFactory = mock( ProgressMonitorFactory.class ); - private Path reportDir = mock( Path.class ); - private ConsistencyCheckService.Result consistencyCheckResult = mock( ConsistencyCheckService.Result.class ); + private final ProgressMonitorFactory progressMonitorFactory = mock( ProgressMonitorFactory.class ); + private final Path reportDir = mock( Path.class ); + private final ConsistencyCheckService.Result consistencyCheckResult = mock( ConsistencyCheckService.Result.class ); @Before public void setup() { + when( outsideWorld.fileSystem() ).thenReturn( fileSystem ); when( onlineBackupContext.getRequiredArguments() ).thenReturn( requiredArguments ); when( requiredArguments.getReportDir() ).thenReturn( reportDir ); subject = new BackupFlow( consistencyCheckService, outsideWorld, logProvider, progressMonitorFactory, diff --git a/enterprise/backup/src/test/java/org/neo4j/backup/BackupIT.java b/enterprise/backup/src/test/java/org/neo4j/backup/BackupIT.java index bc5552589418e..7b0281236dac4 100644 --- a/enterprise/backup/src/test/java/org/neo4j/backup/BackupIT.java +++ b/enterprise/backup/src/test/java/org/neo4j/backup/BackupIT.java @@ -20,6 +20,7 @@ package org.neo4j.backup; import org.apache.commons.io.FileUtils; +import org.hamcrest.Matchers; import org.junit.After; import org.junit.Before; import org.junit.Rule; @@ -66,6 +67,8 @@ import org.neo4j.kernel.impl.store.id.IdGeneratorImpl; import org.neo4j.kernel.impl.storemigration.StoreFileType; import org.neo4j.kernel.impl.transaction.TransactionHeaderInformationFactory; +import org.neo4j.kernel.impl.transaction.log.files.LogFiles; +import org.neo4j.kernel.impl.transaction.log.files.LogFilesBuilder; import org.neo4j.ports.allocation.PortAuthority; import org.neo4j.test.DbRepresentation; import org.neo4j.test.TestGraphDatabaseFactory; @@ -493,6 +496,34 @@ public void shouldLeaveIdFilesAfterBackup() throws Exception } } + @Test + public void backupDatabaseWithCustomTransactionLogsLocation() throws IOException + { + int backupPort = PortAuthority.allocatePort(); + GraphDatabaseService db = startGraphDatabase( serverPath, true, backupPort, "customLogLocation" ); + try + { + createInitialDataset( db ); + + OnlineBackup backup = OnlineBackup.from( "127.0.0.1", backupPort ); + String backupStore = backupPath.getPath(); + LogFiles logFiles = LogFilesBuilder.logFilesBasedOnlyBuilder( new File( backupStore ), fileSystemRule.get() ).build(); + + backup.full( backupStore ); + assertThat( logFiles.logFiles(), Matchers.arrayWithSize( 1 ) ); + + DbRepresentation representation = addLotsOfData( db ); + backup.incremental( backupStore ); + assertThat( logFiles.logFiles(), Matchers.arrayWithSize( 1 ) ); + + assertEquals( representation, getDbRepresentation() ); + } + finally + { + db.shutdown(); + } + } + private void ensureStoresHaveIdFiles( File path ) throws IOException { for ( StoreFile file : StoreFile.values() ) @@ -637,6 +668,12 @@ private DbRepresentation addMoreData( File path ) } private GraphDatabaseService startGraphDatabase( File storeDir, boolean withOnlineBackup, Integer backupPort ) + { + return startGraphDatabase( storeDir, withOnlineBackup, backupPort, "" ); + } + + private GraphDatabaseService startGraphDatabase( File storeDir, boolean withOnlineBackup, Integer backupPort, + String logLocation ) { GraphDatabaseFactory dbFactory = new TestGraphDatabaseFactory() { @@ -668,7 +705,8 @@ protected TransactionHeaderInformation createUsing( byte[] additionalHeader ) GraphDatabaseBuilder graphDatabaseBuilder = dbFactory.newEmbeddedDatabaseBuilder( storeDir ) .setConfig( OnlineBackupSettings.online_backup_enabled, String.valueOf( withOnlineBackup ) ) .setConfig( GraphDatabaseSettings.keep_logical_logs, Settings.TRUE ) - .setConfig( GraphDatabaseSettings.record_format, recordFormatName ); + .setConfig( GraphDatabaseSettings.record_format, recordFormatName ) + .setConfig( GraphDatabaseSettings.logical_logs_location, logLocation ); if ( backupPort != null ) { diff --git a/enterprise/backup/src/test/java/org/neo4j/backup/BackupProtocolServiceIT.java b/enterprise/backup/src/test/java/org/neo4j/backup/BackupProtocolServiceIT.java index 0ae2d19fff55b..8f10f30c720c0 100644 --- a/enterprise/backup/src/test/java/org/neo4j/backup/BackupProtocolServiceIT.java +++ b/enterprise/backup/src/test/java/org/neo4j/backup/BackupProtocolServiceIT.java @@ -1087,7 +1087,7 @@ private void checkLastCommittedTxIdInLogAndNeoStore( long txId, long txIdFromOri LifeSupport life = new LifeSupport(); PageCache pageCache = pageCacheRule.getPageCache( fileSystem ); LogicalTransactionStore transactionStore = - life.add( new ReadOnlyTransactionStore( pageCache, fileSystem, backupDir, monitors ) ); + life.add( new ReadOnlyTransactionStore( pageCache, fileSystem, backupDir, Config.defaults(), monitors ) ); life.start(); try ( IOCursor cursor = transactionStore.getTransactions( txId ) ) diff --git a/enterprise/backup/src/test/java/org/neo4j/backup/OnlineBackupCommandConfigLoaderTest.java b/enterprise/backup/src/test/java/org/neo4j/backup/OnlineBackupCommandConfigLoaderTest.java index dfcdf7e1ca29d..62920e2d57039 100644 --- a/enterprise/backup/src/test/java/org/neo4j/backup/OnlineBackupCommandConfigLoaderTest.java +++ b/enterprise/backup/src/test/java/org/neo4j/backup/OnlineBackupCommandConfigLoaderTest.java @@ -23,7 +23,6 @@ import org.junit.Rule; import org.junit.Test; import org.junit.rules.ExpectedException; -import org.junit.rules.TemporaryFolder; import java.io.BufferedWriter; import java.io.File; @@ -35,7 +34,6 @@ import org.neo4j.causalclustering.core.CausalClusteringSettings; import org.neo4j.commandline.admin.CommandFailed; -import org.neo4j.graphdb.config.InvalidSettingException; import org.neo4j.kernel.configuration.Config; import org.neo4j.test.rule.TestDirectory; diff --git a/enterprise/backup/src/test/java/org/neo4j/causalclustering/BackupCoreIT.java b/enterprise/backup/src/test/java/org/neo4j/causalclustering/BackupCoreIT.java index ee1979c06769a..c755f095411a7 100644 --- a/enterprise/backup/src/test/java/org/neo4j/causalclustering/BackupCoreIT.java +++ b/enterprise/backup/src/test/java/org/neo4j/causalclustering/BackupCoreIT.java @@ -24,27 +24,24 @@ import org.junit.Test; import java.io.File; -import java.net.InetSocketAddress; import java.util.ArrayList; import java.util.List; import java.util.Map; -import org.neo4j.causalclustering.core.CausalClusteringSettings; import org.neo4j.causalclustering.core.CoreGraphDatabase; import org.neo4j.causalclustering.core.consensus.roles.Role; import org.neo4j.causalclustering.discovery.Cluster; import org.neo4j.causalclustering.discovery.CoreClusterMember; import org.neo4j.graphdb.Node; import org.neo4j.graphdb.factory.GraphDatabaseSettings; -import org.neo4j.helpers.HostnamePort; import org.neo4j.helpers.collection.MapUtil; import org.neo4j.kernel.configuration.Config; import org.neo4j.kernel.impl.enterprise.configuration.OnlineBackupSettings; -import org.neo4j.kernel.impl.factory.GraphDatabaseFacade; import org.neo4j.kernel.impl.store.format.standard.Standard; import org.neo4j.ports.allocation.PortAuthority; import org.neo4j.test.DbRepresentation; import org.neo4j.test.causalclustering.ClusterRule; +import org.neo4j.test.rule.SuppressOutput; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotEquals; @@ -54,7 +51,9 @@ public class BackupCoreIT { @Rule - public ClusterRule clusterRule = new ClusterRule( getClass() ) + public final SuppressOutput suppressOutput = SuppressOutput.suppressAll(); + @Rule + public final ClusterRule clusterRule = new ClusterRule( getClass() ) .withNumberOfCoreMembers( 3 ) .withNumberOfReadReplicas( 0 ); diff --git a/enterprise/backup/src/test/java/org/neo4j/causalclustering/ClusterSeedingIT.java b/enterprise/backup/src/test/java/org/neo4j/causalclustering/ClusterSeedingIT.java index 4c22b8c682f71..3c1f42ed64fdc 100644 --- a/enterprise/backup/src/test/java/org/neo4j/causalclustering/ClusterSeedingIT.java +++ b/enterprise/backup/src/test/java/org/neo4j/causalclustering/ClusterSeedingIT.java @@ -32,16 +32,17 @@ import org.neo4j.causalclustering.discovery.CoreClusterMember; import org.neo4j.causalclustering.discovery.IpFamily; import org.neo4j.causalclustering.discovery.SharedDiscoveryService; +import org.neo4j.graphdb.factory.GraphDatabaseSettings; import org.neo4j.io.fs.FileSystemAbstraction; import org.neo4j.kernel.configuration.Config; import org.neo4j.kernel.impl.enterprise.configuration.OnlineBackupSettings; import org.neo4j.kernel.impl.store.format.standard.Standard; +import org.neo4j.restore.RestoreDatabaseCommand; import org.neo4j.test.DbRepresentation; import org.neo4j.test.rule.TestDirectory; import org.neo4j.test.rule.fs.DefaultFileSystemRule; import static java.util.Collections.emptyMap; -import static java.util.Collections.singletonMap; import static org.junit.Assert.assertEquals; import static org.neo4j.causalclustering.BackupCoreIT.backupAddress; import static org.neo4j.causalclustering.discovery.Cluster.dataMatchesEventually; @@ -136,7 +137,8 @@ public void shouldSeedNewMemberFromEmptyIdleCluster() throws Throwable // and: seeding new member with said backup CoreClusterMember newMember = cluster.addCoreMemberWithId( 3 ); - fsa.copyRecursively( backupDir, newMember.storeDir() ); + String databaseName = newMember.getMemberConfig().get( GraphDatabaseSettings.active_database ); + new RestoreDatabaseCommand( fsa, backupDir, newMember.getMemberConfig(), databaseName, true ).execute(); newMember.start(); // then @@ -158,7 +160,8 @@ public void shouldSeedNewMemberFromNonEmptyIdleCluster() throws Throwable // and: seeding new member with said backup CoreClusterMember newMember = cluster.addCoreMemberWithId( 3 ); - fsa.copyRecursively( backupDir, newMember.storeDir() ); + String databaseName = newMember.getMemberConfig().get( GraphDatabaseSettings.active_database ); + new RestoreDatabaseCommand( fsa, backupDir, newMember.getMemberConfig(), databaseName, true ).execute(); newMember.start(); // then diff --git a/enterprise/backup/src/test/java/org/neo4j/restore/RestoreDatabaseCommandIT.java b/enterprise/backup/src/test/java/org/neo4j/restore/RestoreDatabaseCommandIT.java index 272f942873f12..dda5120b588d5 100644 --- a/enterprise/backup/src/test/java/org/neo4j/restore/RestoreDatabaseCommandIT.java +++ b/enterprise/backup/src/test/java/org/neo4j/restore/RestoreDatabaseCommandIT.java @@ -21,33 +21,42 @@ import org.junit.Rule; import org.junit.Test; +import org.mockito.Mockito; import java.io.ByteArrayOutputStream; import java.io.File; +import java.io.IOException; import java.io.PrintStream; +import org.neo4j.commandline.admin.CommandFailed; import org.neo4j.commandline.admin.CommandLocator; import org.neo4j.commandline.admin.Usage; -import org.neo4j.dbms.DatabaseManagementSystemSettings; import org.neo4j.graphdb.GraphDatabaseService; import org.neo4j.graphdb.Transaction; import org.neo4j.graphdb.factory.GraphDatabaseFactory; +import org.neo4j.graphdb.factory.GraphDatabaseSettings; import org.neo4j.helpers.collection.Iterables; import org.neo4j.io.fs.FileSystemAbstraction; import org.neo4j.kernel.configuration.Config; import org.neo4j.kernel.configuration.Settings; import org.neo4j.kernel.impl.enterprise.configuration.OnlineBackupSettings; +import org.neo4j.kernel.impl.transaction.log.files.LogFiles; +import org.neo4j.kernel.impl.transaction.log.files.LogFilesBuilder; import org.neo4j.kernel.internal.locker.StoreLocker; -import org.neo4j.ports.allocation.PortAuthority; import org.neo4j.test.rule.TestDirectory; import org.neo4j.test.rule.fs.DefaultFileSystemRule; +import static org.hamcrest.Matchers.arrayWithSize; +import static org.hamcrest.Matchers.emptyArray; import static org.hamcrest.Matchers.equalTo; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertThat; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; +import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; import static org.neo4j.helpers.collection.MapUtil.stringMap; public class RestoreDatabaseCommandIT @@ -64,7 +73,7 @@ public void forceShouldRespectStoreLock() throws Exception Config config = configWith( databaseName, directory.absolutePath().getAbsolutePath() ); File fromPath = new File( directory.absolutePath(), "from" ); - File toPath = config.get( DatabaseManagementSystemSettings.database_path ); + File toPath = config.get( GraphDatabaseSettings.database_path ); int fromNodeCount = 10; int toNodeCount = 20; @@ -93,7 +102,7 @@ public void shouldNotCopyOverAndExistingDatabase() throws Exception Config config = configWith( databaseName, directory.absolutePath().getAbsolutePath() ); File fromPath = new File( directory.absolutePath(), "from" ); - File toPath = config.get( DatabaseManagementSystemSettings.database_path ); + File toPath = config.get( GraphDatabaseSettings.database_path ); createDbAt( fromPath, 0 ); createDbAt( toPath, 0 ); @@ -121,7 +130,7 @@ public void shouldThrowExceptionIfBackupDirectoryDoesNotExist() throws Exception Config config = configWith( databaseName, directory.absolutePath().getAbsolutePath() ); File fromPath = new File( directory.absolutePath(), "from" ); - File toPath = config.get( DatabaseManagementSystemSettings.database_path ); + File toPath = config.get( GraphDatabaseSettings.database_path ); createDbAt( toPath, 0 ); @@ -171,7 +180,7 @@ public void shouldAllowForcedCopyOverAnExistingDatabase() throws Exception Config config = configWith( databaseName, directory.absolutePath().getAbsolutePath() ); File fromPath = new File( directory.absolutePath(), "from" ); - File toPath = config.get( DatabaseManagementSystemSettings.database_path ); + File toPath = config.get( GraphDatabaseSettings.database_path ); int fromNodeCount = 10; int toNodeCount = 20; @@ -194,6 +203,61 @@ public void shouldAllowForcedCopyOverAnExistingDatabase() throws Exception copiedDb.shutdown(); } + @Test + public void restoreTransactionLogsInCustomDirectoryForTargetDatabaseWhenConfigured() + throws IOException, CommandFailed + { + String databaseName = "to"; + Config config = configWith( databaseName, directory.absolutePath().getAbsolutePath() ); + File customTxLogDirectory = directory.directory( "customLogicalLog" ); + String customTransactionLogDirectory = customTxLogDirectory.getAbsolutePath(); + config.augmentDefaults( GraphDatabaseSettings.logical_logs_location, customTransactionLogDirectory ); + + File fromPath = new File( directory.absolutePath(), "from" ); + File toPath = config.get( GraphDatabaseSettings.database_path ); + int fromNodeCount = 10; + int toNodeCount = 20; + createDbAt( fromPath, fromNodeCount ); + + GraphDatabaseService db = new GraphDatabaseFactory() + .newEmbeddedDatabaseBuilder( toPath ) + .setConfig( OnlineBackupSettings.online_backup_enabled, Settings.FALSE ) + .setConfig( GraphDatabaseSettings.logical_logs_location, customTransactionLogDirectory ) + .newGraphDatabase(); + createTestData( toNodeCount, db ); + db.shutdown(); + + // when + new RestoreDatabaseCommand( fileSystemRule.get(), fromPath, config, databaseName, true ).execute(); + + LogFiles fromStoreLogFiles = LogFilesBuilder.logFilesBasedOnlyBuilder( fromPath, fileSystemRule.get() ).build(); + LogFiles toStoreLogFiles = LogFilesBuilder.logFilesBasedOnlyBuilder( toPath, fileSystemRule.get() ).build(); + LogFiles customLogLocationLogFiles = LogFilesBuilder.logFilesBasedOnlyBuilder( customTxLogDirectory, fileSystemRule.get() ).build(); + assertThat( toStoreLogFiles.logFiles(), emptyArray() ); + assertThat( customLogLocationLogFiles.logFiles(), arrayWithSize( 1 ) ); + assertEquals( fromStoreLogFiles.getLogFileForVersion( 0 ).length(), + customLogLocationLogFiles.getLogFileForVersion( 0 ).length() ); + } + + @Test + public void doNotRemoveRelativeTransactionDirectoryAgain() throws IOException, CommandFailed + { + FileSystemAbstraction fileSystem = Mockito.spy( fileSystemRule.get() ); + File fromPath = directory.directory( "from" ); + File databaseFile = directory.directory(); + File relativeLogDirectory = directory.directory( "relativeDirectory" ); + + Config config = Config.defaults( GraphDatabaseSettings.database_path, databaseFile.getAbsolutePath() ); + config.augment( GraphDatabaseSettings.logical_logs_location, relativeLogDirectory.getAbsolutePath() ); + + createDbAt( fromPath, 10 ); + + new RestoreDatabaseCommand( fileSystem, fromPath, config, "testDatabase", true ).execute(); + + verify( fileSystem ).deleteRecursively( eq( databaseFile ) ); + verify( fileSystem, never() ).deleteRecursively( eq( relativeLogDirectory ) ); + } + @Test public void shouldPrintNiceHelp() throws Throwable { @@ -227,18 +291,24 @@ public void shouldPrintNiceHelp() throws Throwable private static Config configWith( String databaseName, String dataDirectory ) { - return Config.defaults( stringMap( DatabaseManagementSystemSettings.active_database.name(), databaseName, - DatabaseManagementSystemSettings.data_directory.name(), dataDirectory ) ); + return Config.defaults( stringMap( GraphDatabaseSettings.active_database.name(), databaseName, + GraphDatabaseSettings.data_directory.name(), dataDirectory ) ); } private void createDbAt( File fromPath, int nodesToCreate ) { - GraphDatabaseFactory factory = new GraphDatabaseFactory(); - - GraphDatabaseService db = factory.newEmbeddedDatabaseBuilder( fromPath ) - .setConfig( OnlineBackupSettings.online_backup_server, "127.0.0.1:" + PortAuthority.allocatePort() ) + GraphDatabaseService db = new GraphDatabaseFactory().newEmbeddedDatabaseBuilder( fromPath ) + .setConfig( OnlineBackupSettings.online_backup_enabled, Settings.FALSE ) + .setConfig( GraphDatabaseSettings.logical_logs_location, fromPath.getAbsolutePath() ) .newGraphDatabase(); + createTestData( nodesToCreate, db ); + + db.shutdown(); + } + + private void createTestData( int nodesToCreate, GraphDatabaseService db ) + { try ( Transaction tx = db.beginTx() ) { for ( int i = 0; i < nodesToCreate; i++ ) @@ -247,7 +317,5 @@ private void createDbAt( File fromPath, int nodesToCreate ) } tx.success(); } - - db.shutdown(); } } diff --git a/enterprise/causal-clustering/src/main/java/org/neo4j/causalclustering/catchup/storecopy/CopiedStoreRecovery.java b/enterprise/causal-clustering/src/main/java/org/neo4j/causalclustering/catchup/storecopy/CopiedStoreRecovery.java index f78dc5fce4c86..7a2fe16299bf3 100644 --- a/enterprise/causal-clustering/src/main/java/org/neo4j/causalclustering/catchup/storecopy/CopiedStoreRecovery.java +++ b/enterprise/causal-clustering/src/main/java/org/neo4j/causalclustering/catchup/storecopy/CopiedStoreRecovery.java @@ -28,6 +28,7 @@ import org.neo4j.kernel.configuration.Config; import org.neo4j.kernel.configuration.Settings; import org.neo4j.kernel.extension.KernelExtensionFactory; +import org.neo4j.kernel.impl.enterprise.configuration.OnlineBackupSettings; import org.neo4j.kernel.impl.storemigration.UpgradeNotAllowedByConfigurationException; import org.neo4j.kernel.lifecycle.LifecycleAdapter; import org.neo4j.logging.NullLogProvider; @@ -98,8 +99,7 @@ private GraphDatabaseService newTempDatabase( File tempStore ) .setKernelExtensions( kernelExtensions ) .setUserLogProvider( NullLogProvider.getInstance() ) .newEmbeddedDatabaseBuilder( tempStore ) - .setConfig( "dbms.backup.enabled", Settings.FALSE ) - .setConfig( GraphDatabaseSettings.logs_directory, tempStore.getAbsolutePath() ) + .setConfig( OnlineBackupSettings.online_backup_enabled, Settings.FALSE ) .setConfig( GraphDatabaseSettings.keep_logical_logs, Settings.TRUE ) .setConfig( GraphDatabaseSettings.allow_upgrade, config.get( GraphDatabaseSettings.allow_upgrade ).toString() ) diff --git a/enterprise/causal-clustering/src/main/java/org/neo4j/causalclustering/catchup/storecopy/LocalDatabase.java b/enterprise/causal-clustering/src/main/java/org/neo4j/causalclustering/catchup/storecopy/LocalDatabase.java index fb8ff07e2c241..7dbf18370ee48 100644 --- a/enterprise/causal-clustering/src/main/java/org/neo4j/causalclustering/catchup/storecopy/LocalDatabase.java +++ b/enterprise/causal-clustering/src/main/java/org/neo4j/causalclustering/catchup/storecopy/LocalDatabase.java @@ -36,6 +36,7 @@ import org.neo4j.kernel.impl.store.StoreFile; import org.neo4j.kernel.impl.store.StoreType; import org.neo4j.kernel.impl.transaction.log.TransactionAppender; +import org.neo4j.kernel.impl.transaction.log.files.LogFiles; import org.neo4j.kernel.impl.transaction.state.DataSourceManager; import org.neo4j.kernel.impl.util.watcher.FileSystemWatcherService; import org.neo4j.kernel.internal.DatabaseHealth; @@ -67,20 +68,25 @@ public class LocalDatabase implements Lifecycle private volatile AvailabilityRequirement currentRequirement; private volatile TransactionCommitProcess localCommit; - - public LocalDatabase( File storeDir, StoreFiles storeFiles, DataSourceManager dataSourceManager, - Supplier databaseHealthSupplier, FileSystemWatcherService watcherService, + private LogFiles logFiles; + + public LocalDatabase( File storeDir, + StoreFiles storeFiles, + LogFiles logFiles, + DataSourceManager dataSourceManager, + Supplier databaseHealthSupplier, + FileSystemWatcherService watcherService, AvailabilityGuard availabilityGuard, LogProvider logProvider ) { this.storeDir = storeDir; this.storeFiles = storeFiles; + this.logFiles = logFiles; this.dataSourceManager = dataSourceManager; this.databaseHealthSupplier = databaseHealthSupplier; this.availabilityGuard = availabilityGuard; this.watcherService = watcherService; this.log = logProvider.getLog( getClass() ); - raiseAvailabilityGuard( NOT_STOPPED ); } @@ -182,7 +188,7 @@ private DatabaseHealth getDatabaseHealth() public void delete() throws IOException { - storeFiles.delete( storeDir ); + storeFiles.delete( storeDir, logFiles ); } public boolean isEmpty() throws IOException @@ -203,8 +209,8 @@ public File storeDir() void replaceWith( File sourceDir ) throws IOException { - storeFiles.delete( storeDir ); - storeFiles.moveTo( sourceDir, storeDir ); + storeFiles.delete( storeDir, logFiles ); + storeFiles.moveTo( sourceDir, storeDir, logFiles ); } public NeoStoreDataSource dataSource() diff --git a/enterprise/causal-clustering/src/main/java/org/neo4j/causalclustering/catchup/storecopy/RemoteStore.java b/enterprise/causal-clustering/src/main/java/org/neo4j/causalclustering/catchup/storecopy/RemoteStore.java index dddc257dd316c..d754ad7f34647 100644 --- a/enterprise/causal-clustering/src/main/java/org/neo4j/causalclustering/catchup/storecopy/RemoteStore.java +++ b/enterprise/causal-clustering/src/main/java/org/neo4j/causalclustering/catchup/storecopy/RemoteStore.java @@ -32,6 +32,7 @@ import org.neo4j.helpers.AdvertisedSocketAddress; import org.neo4j.io.fs.FileSystemAbstraction; import org.neo4j.io.pagecache.PageCache; +import org.neo4j.kernel.configuration.Config; import org.neo4j.kernel.impl.transaction.CommittedTransactionRepresentation; import org.neo4j.kernel.impl.transaction.log.NoSuchTransactionException; import org.neo4j.kernel.impl.transaction.log.ReadOnlyTransactionIdStore; @@ -53,6 +54,7 @@ public class RemoteStore { private final Log log; + private final Config config; private final Monitors monitors; private final FileSystemAbstraction fs; private final PageCache pageCache; @@ -61,8 +63,8 @@ public class RemoteStore private final TxPullClient txPullClient; private final TransactionLogCatchUpFactory transactionLogFactory; - public RemoteStore( LogProvider logProvider, FileSystemAbstraction fs, PageCache pageCache, StoreCopyClient storeCopyClient, TxPullClient txPullClient, - TransactionLogCatchUpFactory transactionLogFactory, Monitors monitors ) + public RemoteStore( LogProvider logProvider, FileSystemAbstraction fs, PageCache pageCache, StoreCopyClient storeCopyClient, + TxPullClient txPullClient, TransactionLogCatchUpFactory transactionLogFactory, Config config, Monitors monitors ) { this.logProvider = logProvider; this.storeCopyClient = storeCopyClient; @@ -70,6 +72,7 @@ public RemoteStore( LogProvider logProvider, FileSystemAbstraction fs, PageCache this.fs = fs; this.pageCache = pageCache; this.transactionLogFactory = transactionLogFactory; + this.config = config; this.monitors = monitors; this.log = logProvider.getLog( getClass() ); } @@ -97,7 +100,8 @@ private long getPullIndex( File storeDir ) throws IOException log.info( "Last Clean Tx Id: %d", lastCleanTxId ); /* these are the transaction logs */ - ReadOnlyTransactionStore txStore = new ReadOnlyTransactionStore( pageCache, fs, storeDir, new Monitors() ); + ReadOnlyTransactionStore txStore = new ReadOnlyTransactionStore( pageCache, fs, storeDir, config, + new Monitors() ); long lastTxId = BASE_TX_ID; try ( Lifespan ignored = new Lifespan( txStore ); TransactionCursor cursor = txStore.getTransactions( lastCleanTxId ) ) @@ -123,10 +127,11 @@ private long getPullIndex( File storeDir ) throws IOException } } - public CatchupResult tryCatchingUp( AdvertisedSocketAddress from, StoreId expectedStoreId, File storeDir ) throws StoreCopyFailedException, IOException + public CatchupResult tryCatchingUp( AdvertisedSocketAddress from, StoreId expectedStoreId, File storeDir, + boolean keepTxLogsInStoreDir ) throws StoreCopyFailedException, IOException { long pullIndex = getPullIndex( storeDir ); - return pullTransactions( from, expectedStoreId, storeDir, pullIndex, false ); + return pullTransactions( from, expectedStoreId, storeDir, pullIndex, false, keepTxLogsInStoreDir ); } public void copy( AdvertisedSocketAddress from, StoreId expectedStoreId, File destDir ) @@ -143,7 +148,8 @@ public void copy( AdvertisedSocketAddress from, StoreId expectedStoreId, File de log.info( "Store files need to be recovered starting from: %d", lastFlushedTxId ); - CatchupResult catchupResult = pullTransactions( from, expectedStoreId, destDir, lastFlushedTxId, true ); + CatchupResult catchupResult = + pullTransactions( from, expectedStoreId, destDir, lastFlushedTxId, true, true ); if ( catchupResult != SUCCESS_END_OF_STREAM ) { throw new StreamingTransactionsFailedException( "Failed to pull transactions: " + catchupResult ); @@ -155,10 +161,12 @@ public void copy( AdvertisedSocketAddress from, StoreId expectedStoreId, File de } } - private CatchupResult pullTransactions( AdvertisedSocketAddress from, StoreId expectedStoreId, File storeDir, long fromTxId, boolean asPartOfStoreCopy ) + private CatchupResult pullTransactions( AdvertisedSocketAddress from, StoreId expectedStoreId, File storeDir, long fromTxId, + boolean asPartOfStoreCopy, boolean keepTxLogsInStoreDir ) throws IOException, StoreCopyFailedException { - try ( TransactionLogCatchUpWriter writer = transactionLogFactory.create( storeDir, fs, pageCache, logProvider, fromTxId, asPartOfStoreCopy ) ) + try ( TransactionLogCatchUpWriter writer = transactionLogFactory.create( storeDir, fs, pageCache, config, + logProvider, fromTxId, asPartOfStoreCopy, keepTxLogsInStoreDir ) ) { log.info( "Pulling transactions from: %d", fromTxId ); diff --git a/enterprise/causal-clustering/src/main/java/org/neo4j/causalclustering/catchup/storecopy/StoreCopyProcess.java b/enterprise/causal-clustering/src/main/java/org/neo4j/causalclustering/catchup/storecopy/StoreCopyProcess.java index 55f4378ee3698..813fc59d15286 100644 --- a/enterprise/causal-clustering/src/main/java/org/neo4j/causalclustering/catchup/storecopy/StoreCopyProcess.java +++ b/enterprise/causal-clustering/src/main/java/org/neo4j/causalclustering/catchup/storecopy/StoreCopyProcess.java @@ -37,8 +37,8 @@ public class StoreCopyProcess private final Log log; private final RemoteStore remoteStore; - public StoreCopyProcess( FileSystemAbstraction fs, PageCache pageCache, LocalDatabase localDatabase, CopiedStoreRecovery copiedStoreRecovery, - RemoteStore remoteStore, LogProvider logProvider ) + public StoreCopyProcess( FileSystemAbstraction fs, PageCache pageCache, LocalDatabase localDatabase, + CopiedStoreRecovery copiedStoreRecovery, RemoteStore remoteStore, LogProvider logProvider ) { this.fs = fs; this.pageCache = pageCache; diff --git a/enterprise/causal-clustering/src/main/java/org/neo4j/causalclustering/catchup/storecopy/StoreFiles.java b/enterprise/causal-clustering/src/main/java/org/neo4j/causalclustering/catchup/storecopy/StoreFiles.java index 6e7d2dea2c347..09b68f1f76b75 100644 --- a/enterprise/causal-clustering/src/main/java/org/neo4j/causalclustering/catchup/storecopy/StoreFiles.java +++ b/enterprise/causal-clustering/src/main/java/org/neo4j/causalclustering/catchup/storecopy/StoreFiles.java @@ -33,6 +33,7 @@ import org.neo4j.io.fs.FileSystemAbstraction; import org.neo4j.io.pagecache.PageCache; import org.neo4j.kernel.impl.store.MetaDataStore; +import org.neo4j.kernel.impl.transaction.log.files.LogFiles; public class StoreFiles { @@ -60,7 +61,7 @@ public StoreFiles( FileSystemAbstraction fs, PageCache pageCache, FilenameFilter this.fileFilter = fileFilter; } - public void delete( File storeDir ) throws IOException + public void delete( File storeDir, LogFiles logFiles ) throws IOException { // 'files' can be null if the directory doesn't exist. This is fine, we just ignore it then. File[] files = fs.listFiles( storeDir, fileFilter ); @@ -72,6 +73,15 @@ public void delete( File storeDir ) throws IOException } } + File[] txLogs = fs.listFiles( logFiles.logFilesDirectory() ); + if ( txLogs != null ) + { + for ( File txLog : txLogs ) + { + fs.deleteFile( txLog ); + } + } + Iterable iterator = acceptedPageCachedFiles( storeDir )::iterator; for ( FileHandle fh : iterator ) { @@ -94,11 +104,13 @@ private Stream acceptedPageCachedFiles( File storeDir ) throws IOExc } } - public void moveTo( File source, File target ) throws IOException + public void moveTo( File source, File target, LogFiles logFiles ) throws IOException { + fs.mkdirs( logFiles.logFilesDirectory() ); for ( File candidate : fs.listFiles( source, fileFilter ) ) { - fs.moveToDirectory( candidate, target ); + File destination = logFiles.isLogFile( candidate) ? logFiles.logFilesDirectory() : target; + fs.moveToDirectory( candidate, destination ); } Iterable fileHandles = acceptedPageCachedFiles( source )::iterator; diff --git a/enterprise/causal-clustering/src/main/java/org/neo4j/causalclustering/catchup/storecopy/TemporaryStoreDirectory.java b/enterprise/causal-clustering/src/main/java/org/neo4j/causalclustering/catchup/storecopy/TemporaryStoreDirectory.java index ffa8d7cc35ba6..7405c90399caa 100644 --- a/enterprise/causal-clustering/src/main/java/org/neo4j/causalclustering/catchup/storecopy/TemporaryStoreDirectory.java +++ b/enterprise/causal-clustering/src/main/java/org/neo4j/causalclustering/catchup/storecopy/TemporaryStoreDirectory.java @@ -24,6 +24,8 @@ import org.neo4j.io.fs.FileSystemAbstraction; import org.neo4j.io.pagecache.PageCache; +import org.neo4j.kernel.impl.transaction.log.files.LogFiles; +import org.neo4j.kernel.impl.transaction.log.files.LogFilesBuilder; public class TemporaryStoreDirectory implements AutoCloseable { @@ -31,12 +33,14 @@ public class TemporaryStoreDirectory implements AutoCloseable private final File tempStoreDir; private final StoreFiles storeFiles; + private LogFiles tempLogFiles; public TemporaryStoreDirectory( FileSystemAbstraction fs, PageCache pageCache, File parent ) throws IOException { this.tempStoreDir = new File( parent, TEMP_COPY_DIRECTORY_NAME ); storeFiles = new StoreFiles( fs, pageCache, ( directory, name ) -> true ); - storeFiles.delete( tempStoreDir ); + tempLogFiles = LogFilesBuilder.logFilesBasedOnlyBuilder( tempStoreDir, fs ).build(); + storeFiles.delete( tempStoreDir, tempLogFiles ); } public File storeDir() @@ -47,6 +51,6 @@ public File storeDir() @Override public void close() throws IOException { - storeFiles.delete( tempStoreDir ); + storeFiles.delete( tempStoreDir, tempLogFiles ); } } diff --git a/enterprise/causal-clustering/src/main/java/org/neo4j/causalclustering/catchup/tx/TransactionLogCatchUpFactory.java b/enterprise/causal-clustering/src/main/java/org/neo4j/causalclustering/catchup/tx/TransactionLogCatchUpFactory.java index a1ac956bf76ce..818e1980e6a13 100644 --- a/enterprise/causal-clustering/src/main/java/org/neo4j/causalclustering/catchup/tx/TransactionLogCatchUpFactory.java +++ b/enterprise/causal-clustering/src/main/java/org/neo4j/causalclustering/catchup/tx/TransactionLogCatchUpFactory.java @@ -24,13 +24,16 @@ import org.neo4j.io.fs.FileSystemAbstraction; import org.neo4j.io.pagecache.PageCache; +import org.neo4j.kernel.configuration.Config; import org.neo4j.logging.LogProvider; public class TransactionLogCatchUpFactory { public TransactionLogCatchUpWriter create( File storeDir, FileSystemAbstraction fs, PageCache pageCache, - LogProvider logProvider, long fromTxId, boolean asPartOfStoreCopy ) throws IOException + Config config, LogProvider logProvider, long fromTxId, boolean asPartOfStoreCopy, boolean keepTxLogsInStoreDir ) + throws IOException { - return new TransactionLogCatchUpWriter( storeDir, fs, pageCache, logProvider, fromTxId, asPartOfStoreCopy ); + return new TransactionLogCatchUpWriter( storeDir, fs, pageCache, config, logProvider, fromTxId, + asPartOfStoreCopy, keepTxLogsInStoreDir ); } } diff --git a/enterprise/causal-clustering/src/main/java/org/neo4j/causalclustering/catchup/tx/TransactionLogCatchUpWriter.java b/enterprise/causal-clustering/src/main/java/org/neo4j/causalclustering/catchup/tx/TransactionLogCatchUpWriter.java index 4cbc92af6608b..619da11108e43 100644 --- a/enterprise/causal-clustering/src/main/java/org/neo4j/causalclustering/catchup/tx/TransactionLogCatchUpWriter.java +++ b/enterprise/causal-clustering/src/main/java/org/neo4j/causalclustering/catchup/tx/TransactionLogCatchUpWriter.java @@ -24,6 +24,7 @@ import org.neo4j.io.fs.FileSystemAbstraction; import org.neo4j.io.pagecache.PageCache; +import org.neo4j.kernel.configuration.Config; import org.neo4j.kernel.impl.store.MetaDataStore; import org.neo4j.kernel.impl.transaction.CommittedTransactionRepresentation; import org.neo4j.kernel.impl.transaction.log.LogPosition; @@ -52,14 +53,19 @@ public class TransactionLogCatchUpWriter implements TxPullResponseListener, Auto private long lastTxId = -1; private long expectedTxId; - TransactionLogCatchUpWriter( File storeDir, FileSystemAbstraction fs, PageCache pageCache, - LogProvider logProvider, long fromTxId, boolean asPartOfStoreCopy ) throws IOException + TransactionLogCatchUpWriter( File storeDir, FileSystemAbstraction fs, PageCache pageCache, Config config, + LogProvider logProvider, long fromTxId, boolean asPartOfStoreCopy, boolean keepTxLogsInStoreDir ) throws IOException { this.pageCache = pageCache; this.log = logProvider.getLog( getClass() ); this.asPartOfStoreCopy = asPartOfStoreCopy; - this.logFiles = LogFilesBuilder.activeFilesBuilder( storeDir, fs, pageCache ) - .withLastCommittedTransactionIdSupplier( () -> fromTxId - 1 ).build(); + LogFilesBuilder logFilesBuilder = LogFilesBuilder.activeFilesBuilder( storeDir, fs, pageCache ) + .withLastCommittedTransactionIdSupplier( () -> fromTxId - 1 ); + if ( !keepTxLogsInStoreDir ) + { + logFilesBuilder.withConfig( config ); + } + this.logFiles = logFilesBuilder.build(); this.lifespan.add( logFiles ); this.writer = new TransactionLogWriter( new LogEntryWriter( logFiles.getLogFile().getWriter() ) ); this.storeDir = storeDir; diff --git a/enterprise/causal-clustering/src/main/java/org/neo4j/causalclustering/core/EnterpriseCoreEditionModule.java b/enterprise/causal-clustering/src/main/java/org/neo4j/causalclustering/core/EnterpriseCoreEditionModule.java index 10ad2912ba150..d58cdcf27bbcc 100644 --- a/enterprise/causal-clustering/src/main/java/org/neo4j/causalclustering/core/EnterpriseCoreEditionModule.java +++ b/enterprise/causal-clustering/src/main/java/org/neo4j/causalclustering/core/EnterpriseCoreEditionModule.java @@ -22,6 +22,7 @@ import java.io.File; import java.io.FileNotFoundException; import java.io.FileOutputStream; +import java.io.IOException; import java.io.PrintWriter; import java.util.function.Predicate; import java.util.function.Supplier; @@ -63,7 +64,6 @@ import org.neo4j.causalclustering.messaging.RaftOutbound; import org.neo4j.causalclustering.messaging.SenderService; import org.neo4j.com.storecopy.StoreUtil; -import org.neo4j.dbms.DatabaseManagementSystemSettings; import org.neo4j.function.Predicates; import org.neo4j.graphdb.DependencyResolver; import org.neo4j.graphdb.factory.GraphDatabaseSettings; @@ -92,6 +92,8 @@ import org.neo4j.kernel.impl.store.id.IdGeneratorFactory; import org.neo4j.kernel.impl.store.id.IdReuseEligibility; import org.neo4j.kernel.impl.transaction.TransactionHeaderInformationFactory; +import org.neo4j.kernel.impl.transaction.log.files.LogFiles; +import org.neo4j.kernel.impl.transaction.log.files.LogFilesBuilder; import org.neo4j.kernel.impl.transaction.log.files.TransactionLogFiles; import org.neo4j.kernel.impl.util.Dependencies; import org.neo4j.kernel.internal.DatabaseHealth; @@ -102,7 +104,6 @@ import org.neo4j.kernel.lifecycle.LifecycleStatus; import org.neo4j.kernel.monitoring.Monitors; import org.neo4j.logging.LogProvider; -import org.neo4j.ssl.SslPolicy; import org.neo4j.time.Clocks; import org.neo4j.udc.UsageData; @@ -172,7 +173,7 @@ public void registerEditionSpecificProcedures( Procedures procedures ) throws Ke final LifeSupport life = platformModule.life; final Monitors monitors = platformModule.monitors; - final File dataDir = config.get( DatabaseManagementSystemSettings.data_directory ); + final File dataDir = config.get( GraphDatabaseSettings.data_directory ); final ClusterStateDirectory clusterStateDirectory = new ClusterStateDirectory( dataDir, storeDir, false ); try { @@ -192,8 +193,10 @@ public void registerEditionSpecificProcedures( Procedures procedures ) throws Ke watcherService = createFileSystemWatcherService( fileSystem, storeDir, logging, platformModule.jobScheduler, fileWatcherFileNameFilter() ); dependencies.satisfyDependencies( watcherService ); + LogFiles logFiles = buildLocalDatabaseLogFiles( platformModule, fileSystem, storeDir ); LocalDatabase localDatabase = new LocalDatabase( platformModule.storeDir, new StoreFiles( fileSystem, platformModule.pageCache ), + logFiles, platformModule.dataSourceManager, databaseHealthSupplier, watcherService, @@ -264,6 +267,19 @@ public void registerEditionSpecificProcedures( Procedures procedures ) throws Ke life.add( coreServerModule.membershipWaiterLifecycle ); } + private LogFiles buildLocalDatabaseLogFiles( PlatformModule platformModule, FileSystemAbstraction fileSystem, + File storeDir ) + { + try + { + return LogFilesBuilder.activeFilesBuilder( storeDir, fileSystem, platformModule.pageCache ).withConfig( config ).build(); + } + catch ( IOException e ) + { + throw new RuntimeException( e ); + } + } + protected ClusteringModule getClusteringModule( PlatformModule platformModule, DiscoveryServiceFactory discoveryServiceFactory, ClusterStateDirectory clusterStateDirectory, diff --git a/enterprise/causal-clustering/src/main/java/org/neo4j/causalclustering/core/server/CoreServerModule.java b/enterprise/causal-clustering/src/main/java/org/neo4j/causalclustering/core/server/CoreServerModule.java index 70de812ddf123..fa296c20ec64c 100644 --- a/enterprise/causal-clustering/src/main/java/org/neo4j/causalclustering/core/server/CoreServerModule.java +++ b/enterprise/causal-clustering/src/main/java/org/neo4j/causalclustering/core/server/CoreServerModule.java @@ -126,13 +126,13 @@ public CoreServerModule( IdentityModule identityModule, final PlatformModule pla RemoteStore remoteStore = new RemoteStore( logProvider, fileSystem, platformModule.pageCache, new StoreCopyClient( catchUpClient, logProvider ), new TxPullClient( catchUpClient, platformModule.monitors ), new TransactionLogCatchUpFactory(), - platformModule.monitors ); + config, platformModule.monitors ); CopiedStoreRecovery copiedStoreRecovery = new CopiedStoreRecovery( config, platformModule.kernelExtensions.listFactories(), platformModule.pageCache ); life.add( copiedStoreRecovery ); - StoreCopyProcess storeCopyProcess = - new StoreCopyProcess( fileSystem, platformModule.pageCache, localDatabase, copiedStoreRecovery, remoteStore, logProvider ); + StoreCopyProcess storeCopyProcess = new StoreCopyProcess( fileSystem, platformModule.pageCache, localDatabase, + copiedStoreRecovery, remoteStore, logProvider ); LifeSupport servicesToStopOnStoreCopy = new LifeSupport(); diff --git a/enterprise/causal-clustering/src/main/java/org/neo4j/causalclustering/core/state/CoreBootstrapper.java b/enterprise/causal-clustering/src/main/java/org/neo4j/causalclustering/core/state/CoreBootstrapper.java index 8f9a5a963353f..002f25736b713 100644 --- a/enterprise/causal-clustering/src/main/java/org/neo4j/causalclustering/core/state/CoreBootstrapper.java +++ b/enterprise/causal-clustering/src/main/java/org/neo4j/causalclustering/core/state/CoreBootstrapper.java @@ -125,6 +125,7 @@ private void appendNullTransactionLogEntryToSetRaftIndexToMinusOne() throws IOEx { ReadOnlyTransactionIdStore readOnlyTransactionIdStore = new ReadOnlyTransactionIdStore( pageCache, storeDir ); LogFiles logFiles = LogFilesBuilder.activeFilesBuilder( storeDir, fs, pageCache ) + .withConfig( config ) .withLastCommittedTransactionIdSupplier( () -> readOnlyTransactionIdStore.getLastClosedTransactionId() - 1 ) .build(); diff --git a/enterprise/causal-clustering/src/main/java/org/neo4j/causalclustering/core/state/snapshot/CoreStateDownloader.java b/enterprise/causal-clustering/src/main/java/org/neo4j/causalclustering/core/state/snapshot/CoreStateDownloader.java index 54230d982a67d..3675f6f32cb31 100644 --- a/enterprise/causal-clustering/src/main/java/org/neo4j/causalclustering/core/state/snapshot/CoreStateDownloader.java +++ b/enterprise/causal-clustering/src/main/java/org/neo4j/causalclustering/core/state/snapshot/CoreStateDownloader.java @@ -127,7 +127,8 @@ public void onCoreSnapshot( CompletableFuture signal, CoreSnapshot else { StoreId localStoreId = localDatabase.storeId(); - CatchupResult catchupResult = remoteStore.tryCatchingUp( fromAddress, localStoreId, localDatabase.storeDir() ); + CatchupResult catchupResult = remoteStore.tryCatchingUp( fromAddress, localStoreId, localDatabase + .storeDir(), false ); if ( catchupResult == E_TRANSACTION_PRUNED ) { diff --git a/enterprise/causal-clustering/src/main/java/org/neo4j/causalclustering/readreplica/EnterpriseReadReplicaEditionModule.java b/enterprise/causal-clustering/src/main/java/org/neo4j/causalclustering/readreplica/EnterpriseReadReplicaEditionModule.java index 7e1c4e0f74efb..f76fe6f08ea87 100644 --- a/enterprise/causal-clustering/src/main/java/org/neo4j/causalclustering/readreplica/EnterpriseReadReplicaEditionModule.java +++ b/enterprise/causal-clustering/src/main/java/org/neo4j/causalclustering/readreplica/EnterpriseReadReplicaEditionModule.java @@ -20,6 +20,7 @@ package org.neo4j.causalclustering.readreplica; import java.io.File; +import java.io.IOException; import java.util.HashMap; import java.util.Iterator; import java.util.Map; @@ -97,6 +98,8 @@ import org.neo4j.kernel.impl.transaction.log.LogicalTransactionStore; import org.neo4j.kernel.impl.transaction.log.TransactionAppender; import org.neo4j.kernel.impl.transaction.log.TransactionIdStore; +import org.neo4j.kernel.impl.transaction.log.files.LogFiles; +import org.neo4j.kernel.impl.transaction.log.files.LogFilesBuilder; import org.neo4j.kernel.impl.transaction.log.files.TransactionLogFiles; import org.neo4j.kernel.impl.util.Dependencies; import org.neo4j.kernel.internal.DatabaseHealth; @@ -217,14 +220,16 @@ public class EnterpriseReadReplicaEditionModule extends EditionModule DelayedRenewableTimeoutService catchupTimeoutService = new DelayedRenewableTimeoutService( Clocks.systemClock(), logProvider ); StoreFiles storeFiles = new StoreFiles( fileSystem, pageCache ); + LogFiles logFiles = buildLocalDatabaseLogFiles( platformModule, fileSystem, storeDir, config ); LocalDatabase localDatabase = - new LocalDatabase( platformModule.storeDir, storeFiles, platformModule.dataSourceManager, databaseHealthSupplier, watcherService, - platformModule.availabilityGuard, logProvider ); + new LocalDatabase( platformModule.storeDir, storeFiles, logFiles, platformModule.dataSourceManager, + databaseHealthSupplier, + watcherService, platformModule.availabilityGuard, logProvider ); RemoteStore remoteStore = new RemoteStore( platformModule.logging.getInternalLogProvider(), fileSystem, platformModule.pageCache, new StoreCopyClient( catchUpClient, logProvider ), new TxPullClient( catchUpClient, platformModule.monitors ), - new TransactionLogCatchUpFactory(), platformModule.monitors ); + new TransactionLogCatchUpFactory(), config, platformModule.monitors ); CopiedStoreRecovery copiedStoreRecovery = new CopiedStoreRecovery( config, platformModule.kernelExtensions.listFactories(), platformModule.pageCache ); @@ -232,7 +237,8 @@ public class EnterpriseReadReplicaEditionModule extends EditionModule LifeSupport servicesToStopOnStoreCopy = new LifeSupport(); - StoreCopyProcess storeCopyProcess = new StoreCopyProcess( fileSystem, pageCache, localDatabase, copiedStoreRecovery, remoteStore, logProvider ); + StoreCopyProcess storeCopyProcess = new StoreCopyProcess( fileSystem, pageCache, localDatabase, + copiedStoreRecovery, remoteStore, logProvider ); ConnectToRandomCoreServerStrategy defaultStrategy = new ConnectToRandomCoreServerStrategy(); defaultStrategy.inject( topologyService, config, logProvider, myself ); @@ -375,4 +381,17 @@ private static Map backupDisabledSettings() overrideBackupSettings.put( OnlineBackupSettings.online_backup_enabled.name(), Settings.FALSE ); return overrideBackupSettings; } + + private LogFiles buildLocalDatabaseLogFiles( PlatformModule platformModule, FileSystemAbstraction fileSystem, + File storeDir, Config config ) + { + try + { + return LogFilesBuilder.activeFilesBuilder( storeDir, fileSystem, platformModule.pageCache ).withConfig( config ).build(); + } + catch ( IOException e ) + { + throw new RuntimeException( e ); + } + } } diff --git a/enterprise/causal-clustering/src/main/java/org/neo4j/commandline/dbms/UnbindFromClusterCommand.java b/enterprise/causal-clustering/src/main/java/org/neo4j/commandline/dbms/UnbindFromClusterCommand.java index 034a87226ffc0..5dd022531b7fe 100644 --- a/enterprise/causal-clustering/src/main/java/org/neo4j/commandline/dbms/UnbindFromClusterCommand.java +++ b/enterprise/causal-clustering/src/main/java/org/neo4j/commandline/dbms/UnbindFromClusterCommand.java @@ -31,7 +31,7 @@ import org.neo4j.commandline.admin.IncorrectUsage; import org.neo4j.commandline.admin.OutsideWorld; import org.neo4j.commandline.arguments.Arguments; -import org.neo4j.dbms.DatabaseManagementSystemSettings; +import org.neo4j.graphdb.factory.GraphDatabaseSettings; import org.neo4j.io.fs.FileUtils; import org.neo4j.kernel.StoreLockException; import org.neo4j.kernel.configuration.Config; @@ -61,7 +61,7 @@ static Arguments arguments() private static Config loadNeo4jConfig( Path homeDir, Path configDir, String databaseName ) { return fromFile( configDir.resolve( Config.DEFAULT_CONFIG_FILE_NAME ) ) - .withSetting( DatabaseManagementSystemSettings.active_database, databaseName ) + .withSetting( GraphDatabaseSettings.active_database, databaseName ) .withHome( homeDir ).build(); } @@ -71,8 +71,8 @@ public void execute( String[] args ) throws IncorrectUsage, CommandFailed try { Config config = loadNeo4jConfig( homeDir, configDir, arguments.parse( args ).get( "database" ) ); - File dataDirectory = config.get( DatabaseManagementSystemSettings.data_directory ); - Path pathToSpecificDatabase = config.get( DatabaseManagementSystemSettings.database_path ).toPath(); + File dataDirectory = config.get( GraphDatabaseSettings.data_directory ); + Path pathToSpecificDatabase = config.get( GraphDatabaseSettings.database_path ).toPath(); boolean hasDatabase = true; try diff --git a/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/backup/RestoreClusterUtils.java b/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/backup/RestoreClusterUtils.java index 7a801fdf709ac..fd2ed23d76ab2 100644 --- a/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/backup/RestoreClusterUtils.java +++ b/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/backup/RestoreClusterUtils.java @@ -19,6 +19,8 @@ */ package org.neo4j.causalclustering.backup; +import org.apache.commons.lang3.StringUtils; + import java.io.File; import org.neo4j.graphdb.GraphDatabaseService; @@ -37,14 +39,22 @@ private RestoreClusterUtils() { } - public static File createClassicNeo4jStore( File base, FileSystemAbstraction fileSystem, int nodesToCreate, String recordFormat ) + public static File createClassicNeo4jStore( File base, FileSystemAbstraction fileSystem, int nodesToCreate, + String recordFormat ) + { + return createClassicNeo4jStore( base, fileSystem, nodesToCreate, recordFormat, StringUtils.EMPTY ); + } + + public static File createClassicNeo4jStore( File base, FileSystemAbstraction fileSystem, + int nodesToCreate, String recordFormat, String logicalLogsLocation ) { - File existingDbDir = new File( base, "existing" ); + File storeDir = new File( base, "existing" ); GraphDatabaseService db = new TestGraphDatabaseFactory() .setFileSystem( fileSystem ) - .newEmbeddedDatabaseBuilder( existingDbDir ) + .newEmbeddedDatabaseBuilder( storeDir ) .setConfig( GraphDatabaseSettings.record_format, recordFormat ) .setConfig( OnlineBackupSettings.online_backup_enabled, Boolean.FALSE.toString() ) + .setConfig( GraphDatabaseSettings.logical_logs_location, logicalLogsLocation ) .newGraphDatabase(); for ( int i = 0; i < (nodesToCreate / 2); i++ ) @@ -60,6 +70,6 @@ public static File createClassicNeo4jStore( File base, FileSystemAbstraction fil db.shutdown(); - return existingDbDir; + return storeDir; } } diff --git a/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/catchup/storecopy/CopiedStoreRecoveryTest.java b/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/catchup/storecopy/CopiedStoreRecoveryTest.java index ee8d1b5c0929f..39f255c4231f7 100644 --- a/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/catchup/storecopy/CopiedStoreRecoveryTest.java +++ b/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/catchup/storecopy/CopiedStoreRecoveryTest.java @@ -22,6 +22,7 @@ import org.junit.Test; import java.io.File; +import java.io.IOException; import org.neo4j.helpers.collection.Iterables; import org.neo4j.io.pagecache.PageCache; @@ -34,7 +35,7 @@ public class CopiedStoreRecoveryTest { @Test - public void shouldThrowIfAlreadyShutdown() + public void shouldThrowIfAlreadyShutdown() throws IOException { // Given CopiedStoreRecovery copiedStoreRecovery = diff --git a/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/catchup/storecopy/LocalDatabaseTest.java b/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/catchup/storecopy/LocalDatabaseTest.java index 1cb11827738a8..0cf9f55c6f998 100644 --- a/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/catchup/storecopy/LocalDatabaseTest.java +++ b/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/catchup/storecopy/LocalDatabaseTest.java @@ -26,6 +26,7 @@ import java.time.Clock; import org.neo4j.kernel.AvailabilityGuard; +import org.neo4j.kernel.impl.transaction.log.files.LogFiles; import org.neo4j.kernel.impl.transaction.state.DataSourceManager; import org.neo4j.kernel.impl.util.watcher.FileSystemWatcherService; import org.neo4j.kernel.internal.DatabaseHealth; @@ -167,7 +168,7 @@ private static LocalDatabase newLocalDatabase( AvailabilityGuard availabilityGua private static LocalDatabase newLocalDatabase( AvailabilityGuard availabilityGuard, DataSourceManager dataSourceManager, FileSystemWatcherService fileWatcher ) { - return new LocalDatabase( new File( "." ), mock( StoreFiles.class ), dataSourceManager, + return new LocalDatabase( new File( "." ), mock( StoreFiles.class ), mock( LogFiles.class ), dataSourceManager, () -> mock( DatabaseHealth.class ), fileWatcher, availabilityGuard, NullLogProvider.getInstance() ); } diff --git a/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/catchup/storecopy/RemoteStoreTest.java b/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/catchup/storecopy/RemoteStoreTest.java index 758ce49cf4fb7..9cd41137a9f33 100644 --- a/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/catchup/storecopy/RemoteStoreTest.java +++ b/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/catchup/storecopy/RemoteStoreTest.java @@ -31,6 +31,7 @@ import org.neo4j.causalclustering.identity.StoreId; import org.neo4j.helpers.AdvertisedSocketAddress; import org.neo4j.io.fs.FileSystemAbstraction; +import org.neo4j.kernel.configuration.Config; import org.neo4j.kernel.monitoring.Monitors; import org.neo4j.logging.LogProvider; import org.neo4j.logging.NullLogProvider; @@ -60,7 +61,7 @@ public void shouldCopyStoreFilesAndPullTransactions() throws Exception TransactionLogCatchUpWriter writer = mock( TransactionLogCatchUpWriter.class ); RemoteStore remoteStore = new RemoteStore( NullLogProvider.getInstance(), mock( FileSystemAbstraction.class ), - null, storeCopyClient, txPullClient, factory( writer ), new Monitors() ); + null, storeCopyClient, txPullClient, factory( writer ), Config.defaults(), new Monitors() ); // when AdvertisedSocketAddress localhost = new AdvertisedSocketAddress( "127.0.0.1", 1234 ); @@ -90,7 +91,7 @@ public void shouldSetLastPulledTransactionId() throws Exception TransactionLogCatchUpWriter writer = mock( TransactionLogCatchUpWriter.class ); RemoteStore remoteStore = new RemoteStore( NullLogProvider.getInstance(), mock( FileSystemAbstraction.class ), - null, storeCopyClient, txPullClient, factory( writer ), new Monitors() ); + null, storeCopyClient, txPullClient, factory( writer ), Config.defaults(), new Monitors() ); // when remoteStore.copy( localhost, wantedStoreId, new File( "destination" ) ); @@ -112,7 +113,7 @@ public void shouldCloseDownTxLogWriterIfTxStreamingFails() throws Exception RemoteStore remoteStore = new RemoteStore( NullLogProvider.getInstance(), mock( FileSystemAbstraction.class ), null, - storeCopyClient, txPullClient, factory( writer ), new Monitors() ); + storeCopyClient, txPullClient, factory( writer ), Config.defaults(), new Monitors() ); doThrow( StoreCopyFailedException.class ).when( txPullClient ) .pullTransactions( isNull(), eq( storeId ), anyLong(), any() ); @@ -134,8 +135,8 @@ public void shouldCloseDownTxLogWriterIfTxStreamingFails() throws Exception private TransactionLogCatchUpFactory factory( TransactionLogCatchUpWriter writer ) throws IOException { TransactionLogCatchUpFactory factory = mock( TransactionLogCatchUpFactory.class ); - when( factory.create( isNull(), any( FileSystemAbstraction.class ), - isNull(), any( LogProvider.class ), anyLong(), anyBoolean() ) ).thenReturn( writer ); + when( factory.create( isNull(), any( FileSystemAbstraction.class ), isNull(), any( Config.class ), + any( LogProvider.class ), anyLong(), anyBoolean(), anyBoolean() ) ).thenReturn( writer ); return factory; } } diff --git a/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/catchup/storecopy/StoreFilesTest.java b/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/catchup/storecopy/StoreFilesTest.java index f2126298fbf01..f16db942b31a7 100644 --- a/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/catchup/storecopy/StoreFilesTest.java +++ b/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/catchup/storecopy/StoreFilesTest.java @@ -40,6 +40,8 @@ import org.neo4j.io.pagecache.PageCache; import org.neo4j.kernel.impl.store.MetaDataStore; import org.neo4j.kernel.impl.store.MetaDataStore.Position; +import org.neo4j.kernel.impl.transaction.log.files.LogFiles; +import org.neo4j.kernel.impl.transaction.log.files.LogFilesBuilder; import org.neo4j.test.rule.PageCacheRule; import org.neo4j.test.rule.TestDirectory; import org.neo4j.test.rule.fs.EphemeralFileSystemRule; @@ -67,6 +69,7 @@ public class StoreFilesTest private EphemeralFileSystemAbstraction pc; private PageCache pageCache; private StoreFiles storeFiles; + private LogFiles logFiles; public StoreFilesTest() { @@ -93,6 +96,7 @@ public void setUp() throws Exception pc = hiddenFileSystemRule.get(); pageCache = pageCacheRule.getPageCache( pc ); storeFiles = new StoreFiles( fs, pageCache ); + logFiles = LogFilesBuilder.logFilesBasedOnlyBuilder( testDirectory.directory(), fs ).build(); } private void createOnFileSystem( File file ) throws IOException @@ -128,7 +132,7 @@ public void deleteMustRecursivelyRemoveFilesInGivenDirectory() throws Exception assertTrue( fs.fileExists( a ) ); assertTrue( pc.fileExists( b ) ); - storeFiles.delete( dir ); + storeFiles.delete( dir, logFiles ); assertFalse( fs.fileExists( a ) ); assertFalse( pc.fileExists( b ) ); @@ -150,7 +154,7 @@ public void deleteMustNotDeleteIgnoredFiles() throws Exception FilenameFilter filter = ( directory, name ) -> !name.equals( "c" ) && !name.equals( "d" ); storeFiles = new StoreFiles( fs, pageCache, filter ); - storeFiles.delete( dir ); + storeFiles.delete( dir, logFiles ); assertFalse( fs.fileExists( a ) ); assertFalse( pc.fileExists( b ) ); @@ -175,7 +179,7 @@ public void deleteMustNotDeleteFilesInIgnoredDirectories() throws Exception FilenameFilter filter = ( directory, name ) -> !name.startsWith( "ignore" ); storeFiles = new StoreFiles( fs, pageCache, filter ); - storeFiles.delete( dir ); + storeFiles.delete( dir, logFiles ); assertFalse( fs.fileExists( a ) ); assertFalse( pc.fileExists( b ) ); @@ -189,7 +193,7 @@ public void deleteMustSilentlyIgnoreMissingDirectories() throws Exception File dir = getBaseDir(); File sub = new File( dir, "sub" ); - storeFiles.delete( sub ); + storeFiles.delete( sub, logFiles ); } @Test @@ -208,7 +212,7 @@ public void mustMoveFilesToTargetDirectory() throws Exception createOnFileSystem( new File( tgt, ".fs-ignore" ) ); createOnPageCache( new File( tgt, ".pc-ignore" ) ); - storeFiles.moveTo( src, tgt ); + storeFiles.moveTo( src, tgt, logFiles ); assertFalse( fs.fileExists( a ) ); assertFalse( pc.fileExists( b ) ); @@ -233,7 +237,7 @@ public void movedFilesMustRetainTheirRelativePaths() throws Exception createOnFileSystem( new File( tgt, ".fs-ignore" ) ); createOnPageCache( new File( tgt, ".pc-ignore" ) ); - storeFiles.moveTo( src, tgt ); + storeFiles.moveTo( src, tgt, logFiles ); assertFalse( fs.fileExists( a ) ); assertFalse( pc.fileExists( b ) ); @@ -264,7 +268,7 @@ public void moveMustIgnoreFilesFilteredOut() throws Exception FilenameFilter filter = ( directory, name ) -> !name.startsWith( "ignore" ); storeFiles = new StoreFiles( fs, pageCache, filter ); - storeFiles.moveTo( src, tgt ); + storeFiles.moveTo( src, tgt, logFiles ); assertFalse( fs.fileExists( a ) ); assertFalse( pc.fileExists( b ) ); diff --git a/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/catchup/tx/TransactionLogCatchUpWriterTest.java b/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/catchup/tx/TransactionLogCatchUpWriterTest.java index 332e54cad6a93..28569a0044844 100644 --- a/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/catchup/tx/TransactionLogCatchUpWriterTest.java +++ b/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/catchup/tx/TransactionLogCatchUpWriterTest.java @@ -22,14 +22,20 @@ import org.junit.Before; import org.junit.Rule; import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; import java.io.File; import java.io.IOException; +import java.util.Arrays; +import java.util.List; import org.neo4j.causalclustering.identity.StoreId; +import org.neo4j.graphdb.factory.GraphDatabaseSettings; import org.neo4j.io.fs.FileSystemAbstraction; import org.neo4j.io.pagecache.PageCache; import org.neo4j.kernel.NeoStoreDataSource; +import org.neo4j.kernel.configuration.Config; import org.neo4j.kernel.impl.storageengine.impl.recordstorage.RecordStorageCommandReaderFactory; import org.neo4j.kernel.impl.transaction.CommittedTransactionRepresentation; import org.neo4j.kernel.impl.transaction.command.Commands; @@ -64,24 +70,31 @@ import static org.junit.Assert.assertTrue; import static org.neo4j.kernel.impl.transaction.command.Commands.createNode; +@RunWith( Parameterized.class ) public class TransactionLogCatchUpWriterTest { @Rule - public final TestDirectory dir = TestDirectory.testDirectory( getClass() ); - + public final TestDirectory dir = TestDirectory.testDirectory(); @Rule public final DefaultFileSystemRule fsRule = new DefaultFileSystemRule(); - @Rule public final PageCacheRule pageCacheRule = new PageCacheRule(); - @Rule public NeoStoreDataSourceRule dsRule = new NeoStoreDataSourceRule(); + @Parameterized.Parameter + public boolean partOfStoreCopy; + private PageCache pageCache; private FileSystemAbstraction fs; private File storeDir; + @Parameterized.Parameters + public static List partOfStoreCopy() + { + return Arrays.asList( Boolean.TRUE, Boolean.FALSE ); + } + @Before public void setup() throws IOException { @@ -93,14 +106,25 @@ public void setup() throws IOException @Test public void shouldCreateTransactionLogWithCheckpoint() throws Exception { - // given + createTransactionLogWithCheckpoint( Config.defaults(), true ); + } + + @Test + public void createTransactionLogWithCheckpointInCustomLocation() throws IOException + { + createTransactionLogWithCheckpoint( Config.defaults( GraphDatabaseSettings.logical_logs_location, + "custom-tx-logs"), false ); + } + + private void createTransactionLogWithCheckpoint( Config config, boolean logsInStoreDir ) throws IOException + { org.neo4j.kernel.impl.store.StoreId storeId = simulateStoreCopy(); int fromTxId = 37; int endTxId = fromTxId + 5; - TransactionLogCatchUpWriter catchUpWriter = new TransactionLogCatchUpWriter( storeDir, fs, pageCache, - NullLogProvider.getInstance(), fromTxId, true ); + TransactionLogCatchUpWriter catchUpWriter = new TransactionLogCatchUpWriter( storeDir, fs, pageCache, config, + NullLogProvider.getInstance(), fromTxId, partOfStoreCopy, logsInStoreDir ); // when for ( int i = fromTxId; i <= endTxId; i++ ) @@ -111,15 +135,24 @@ public void shouldCreateTransactionLogWithCheckpoint() throws Exception catchUpWriter.close(); // then - verifyTransactionsInLog( fromTxId, endTxId ); - verifyCheckpointInLog(); // necessary for recovery + LogFilesBuilder logFilesBuilder = LogFilesBuilder.activeFilesBuilder( storeDir, fs, pageCache ); + if ( !logsInStoreDir ) + { + logFilesBuilder.withConfig( config ); + } + LogFiles logFiles = logFilesBuilder.build(); + + verifyTransactionsInLog( logFiles, fromTxId, endTxId ); + if ( partOfStoreCopy ) + { + verifyCheckpointInLog( logFiles ); + } } - private void verifyCheckpointInLog() throws IOException + private void verifyCheckpointInLog( LogFiles logFiles ) { LogEntryReader logEntryReader = new VersionAwareLogEntryReader<>( new RecordStorageCommandReaderFactory(), InvalidLogEntryHandler.STRICT ); - LogFiles logFiles = LogFilesBuilder.activeFilesBuilder( storeDir, fs, pageCache ).withLogEntryReader( logEntryReader ).build(); final LogTailScanner logTailScanner = new LogTailScanner( logFiles, logEntryReader, new Monitors() ); LogTailInformation tailInformation = logTailScanner.getTailInformation(); @@ -127,10 +160,10 @@ private void verifyCheckpointInLog() throws IOException assertTrue( tailInformation.commitsAfterLastCheckpoint() ); } - private void verifyTransactionsInLog( long fromTxId, long endTxId ) throws IOException + private void verifyTransactionsInLog( LogFiles logFiles, long fromTxId, long endTxId ) throws + IOException { long expectedTxId = fromTxId; - LogFiles logFiles = LogFilesBuilder.activeFilesBuilder( storeDir, fs, pageCache ).build(); LogVersionedStoreChannel versionedStoreChannel = logFiles.openForVersion( 0 ); try ( ReadableLogChannel channel = new ReadAheadLogChannel( versionedStoreChannel, LogVersionBridge.NO_MORE_CHANNELS, 1024 ) ) diff --git a/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/core/state/CoreBootstrapperIT.java b/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/core/state/CoreBootstrapperIT.java index 9873f5e597ab2..e93188dc449d8 100644 --- a/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/core/state/CoreBootstrapperIT.java +++ b/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/core/state/CoreBootstrapperIT.java @@ -24,6 +24,7 @@ import org.junit.rules.RuleChain; import java.io.File; +import java.io.IOException; import java.util.Set; import org.neo4j.causalclustering.backup.RestoreClusterUtils; @@ -35,6 +36,7 @@ import org.neo4j.causalclustering.core.state.snapshot.CoreStateType; import org.neo4j.causalclustering.core.state.snapshot.RaftCoreState; import org.neo4j.causalclustering.identity.MemberId; +import org.neo4j.graphdb.factory.GraphDatabaseSettings; import org.neo4j.io.fs.FileSystemAbstraction; import org.neo4j.io.pagecache.PageCache; import org.neo4j.kernel.configuration.Config; @@ -48,15 +50,13 @@ import org.neo4j.test.rule.TestDirectory; import org.neo4j.test.rule.fs.DefaultFileSystemRule; +import static java.lang.Integer.parseInt; +import static java.util.UUID.randomUUID; import static org.hamcrest.Matchers.allOf; import static org.hamcrest.Matchers.greaterThanOrEqualTo; import static org.hamcrest.Matchers.lessThanOrEqualTo; - -import static java.lang.Integer.parseInt; -import static java.util.UUID.randomUUID; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertThat; - import static org.neo4j.graphdb.factory.GraphDatabaseSettings.record_id_batch_size; import static org.neo4j.helpers.collection.Iterators.asSet; @@ -82,7 +82,32 @@ public void shouldSetAllCoreState() throws Exception PageCache pageCache = pageCacheRule.getPageCache( fileSystem ); CoreBootstrapper bootstrapper = new CoreBootstrapper( classicNeo4jStore, pageCache, fileSystem, Config.defaults(), NullLogProvider.getInstance() ); + bootstrapAndVerify( nodeCount, fileSystem, classicNeo4jStore, pageCache, Config.defaults(), bootstrapper ); + } + @Test + public void setAllCoreStateOnDatabaseWithCustomLogFilesLocation() throws Exception + { + // given + int nodeCount = 100; + FileSystemAbstraction fileSystem = fileSystemRule.get(); + File baseDirectory = testDirectory.directory(); + String customTransactionLogsLocation = "transaction-logs"; + File classicNeo4jStore = RestoreClusterUtils.createClassicNeo4jStore( baseDirectory, fileSystem, nodeCount, + Standard.LATEST_NAME, customTransactionLogsLocation ); + + PageCache pageCache = pageCacheRule.getPageCache( fileSystem ); + Config config = Config.defaults( GraphDatabaseSettings.logical_logs_location, + customTransactionLogsLocation ); + CoreBootstrapper bootstrapper = new CoreBootstrapper( classicNeo4jStore, pageCache, fileSystem, + config, NullLogProvider.getInstance() ); + + bootstrapAndVerify( nodeCount, fileSystem, classicNeo4jStore, pageCache, config, bootstrapper ); + } + + private void bootstrapAndVerify( long nodeCount, FileSystemAbstraction fileSystem, File classicNeo4jStore, + PageCache pageCache, Config config, CoreBootstrapper bootstrapper ) throws IOException + { // when Set membership = asSet( randomMember(), randomMember(), randomMember() ); CoreSnapshot snapshot = bootstrapper.bootstrap( membership ); @@ -90,7 +115,7 @@ public void shouldSetAllCoreState() throws Exception // then int recordIdBatchSize = parseInt( record_id_batch_size.getDefaultValue() ); assertThat( ((IdAllocationState) snapshot.get( CoreStateType.ID_ALLOCATION )).firstUnallocated( IdType.NODE ), - allOf( greaterThanOrEqualTo( (long) nodeCount ), lessThanOrEqualTo( (long) nodeCount + recordIdBatchSize ) ) ); + allOf( greaterThanOrEqualTo( nodeCount ), lessThanOrEqualTo( nodeCount + recordIdBatchSize ) ) ); /* Bootstrapped state is created in RAFT land at index -1 and term -1. */ assertEquals( 0, snapshot.prevIndex() ); @@ -105,10 +130,11 @@ public void shouldSetAllCoreState() throws Exception /* The session state is initially empty. */ assertEquals( new GlobalSessionTrackerState(), snapshot.get( CoreStateType.SESSION_TRACKER ) ); + ReadOnlyTransactionStore transactionStore = new ReadOnlyTransactionStore( pageCache, fileSystem, + classicNeo4jStore, config, new Monitors() ); LastCommittedIndexFinder lastCommittedIndexFinder = new LastCommittedIndexFinder( new ReadOnlyTransactionIdStore( pageCache, classicNeo4jStore ), - new ReadOnlyTransactionStore( pageCache, fileSystem, classicNeo4jStore, new Monitors() ), - NullLogProvider.getInstance() ); + transactionStore, NullLogProvider.getInstance() ); long lastCommittedIndex = lastCommittedIndexFinder.getLastCommittedIndex(); assertEquals( -1, lastCommittedIndex ); diff --git a/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/core/state/machines/id/RebuildReplicatedIdGeneratorsTest.java b/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/core/state/machines/id/RebuildReplicatedIdGeneratorsTest.java index bf053a0af7dd8..f095b4d920af8 100644 --- a/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/core/state/machines/id/RebuildReplicatedIdGeneratorsTest.java +++ b/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/core/state/machines/id/RebuildReplicatedIdGeneratorsTest.java @@ -90,7 +90,6 @@ public void rebuildReplicatedIdGeneratorsOnRecovery() throws Exception reopenedStores.makeStoreOk(); assertEquals( 51L, reopenedStores.getNodeStore().nextId() ); } - } private ReplicatedIdGeneratorFactory getIdGenerationFactory( FileSystemAbstraction fileSystemAbstraction ) diff --git a/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/core/state/snapshot/CoreStateDownloaderTest.java b/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/core/state/snapshot/CoreStateDownloaderTest.java index 88c81547f060d..60f548a1bf84a 100644 --- a/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/core/state/snapshot/CoreStateDownloaderTest.java +++ b/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/core/state/snapshot/CoreStateDownloaderTest.java @@ -44,6 +44,7 @@ import static org.junit.Assert.fail; import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyBoolean; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.never; import static org.mockito.Mockito.verify; @@ -95,7 +96,7 @@ public void shouldDownloadCompleteStoreWhenEmpty() throws Throwable downloader.downloadSnapshot( remoteMember ); // then - verify( remoteStore, never() ).tryCatchingUp( any(), any(), any() ); + verify( remoteStore, never() ).tryCatchingUp( any(), any(), any(), anyBoolean() ); verify( storeCopyProcess ).replaceWithStoreFrom( remoteAddress, remoteStoreId ); } @@ -136,7 +137,7 @@ public void shouldNotOverwriteNonEmptyMismatchingStore() throws Exception // then verify( remoteStore, never() ).copy( any(), any(), any() ); - verify( remoteStore, never() ).tryCatchingUp( any(), any(), any() ); + verify( remoteStore, never() ).tryCatchingUp( any(), any(), any(), anyBoolean() ); } @Test @@ -145,13 +146,14 @@ public void shouldCatchupIfPossible() throws Exception // given when( localDatabase.isEmpty() ).thenReturn( false ); when( remoteStore.getStoreId( remoteAddress ) ).thenReturn( storeId ); - when( remoteStore.tryCatchingUp( remoteAddress, storeId, storeDir ) ).thenReturn( SUCCESS_END_OF_STREAM ); + when( remoteStore.tryCatchingUp( remoteAddress, storeId, storeDir, false ) ) + .thenReturn( SUCCESS_END_OF_STREAM ); // when downloader.downloadSnapshot( remoteMember ); // then - verify( remoteStore ).tryCatchingUp( remoteAddress, storeId, storeDir ); + verify( remoteStore ).tryCatchingUp( remoteAddress, storeId, storeDir, false ); verify( remoteStore, never() ).copy( any(), any(), any() ); } @@ -161,13 +163,13 @@ public void shouldDownloadWholeStoreIfCannotCatchUp() throws Exception // given when( localDatabase.isEmpty() ).thenReturn( false ); when( remoteStore.getStoreId( remoteAddress ) ).thenReturn( storeId ); - when( remoteStore.tryCatchingUp( remoteAddress, storeId, storeDir ) ).thenReturn( E_TRANSACTION_PRUNED ); + when( remoteStore.tryCatchingUp( remoteAddress, storeId, storeDir, false ) ).thenReturn( E_TRANSACTION_PRUNED ); // when downloader.downloadSnapshot( remoteMember ); // then - verify( remoteStore ).tryCatchingUp( remoteAddress, storeId, storeDir ); + verify( remoteStore ).tryCatchingUp( remoteAddress, storeId, storeDir, false ); verify( storeCopyProcess ).replaceWithStoreFrom( remoteAddress, storeId ); } } diff --git a/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/discovery/CoreClusterMember.java b/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/discovery/CoreClusterMember.java index 1b1d39d0f0255..ce3c86c6c2d50 100644 --- a/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/discovery/CoreClusterMember.java +++ b/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/discovery/CoreClusterMember.java @@ -67,6 +67,7 @@ public class CoreClusterMember implements ClusterMember private final String boltAdvertisedSocketAddress; private final int discoveryPort; protected CoreGraphDatabase database; + private final Config memberConfig; public CoreClusterMember( int serverId, int discoveryPort, @@ -124,12 +125,15 @@ public CoreClusterMember( int serverId, this.neo4jHome = new File( parentDir, "server-core-" + serverId ); config.put( GraphDatabaseSettings.neo4j_home.name(), neo4jHome.getAbsolutePath() ); config.put( GraphDatabaseSettings.logs_directory.name(), new File( neo4jHome, "logs" ).getAbsolutePath() ); + config.put( GraphDatabaseSettings.logical_logs_location.name(), "core-tx-logs-" + serverId ); this.discoveryServiceFactory = discoveryServiceFactory; File dataDir = new File( neo4jHome, "data" ); clusterStateDir = ClusterStateDirectory.withoutInitializing( dataDir ).get(); raftLogDir = new File( clusterStateDir, RAFT_LOG_DIRECTORY_NAME ); storeDir = new File( new File( dataDir, "databases" ), "graph.db" ); + memberConfig = Config.defaults( config ); + //noinspection ResultOfMethodCallIgnored storeDir.mkdirs(); } @@ -152,7 +156,7 @@ public String directURI() @Override public void start() { - database = new CoreGraphDatabase( storeDir, Config.defaults( config ), + database = new CoreGraphDatabase( storeDir, memberConfig, GraphDatabaseDependencies.newDependencies(), discoveryServiceFactory ); } @@ -177,6 +181,11 @@ public File storeDir() return storeDir; } + public Config getMemberConfig() + { + return memberConfig; + } + public RaftLogPruner raftLogPruner() { return database.getDependencyResolver().resolveDependency( RaftLogPruner.class ); diff --git a/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/discovery/ReadReplica.java b/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/discovery/ReadReplica.java index 8e6a9a8c186ac..39badd957129f 100644 --- a/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/discovery/ReadReplica.java +++ b/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/discovery/ReadReplica.java @@ -97,9 +97,11 @@ public ReadReplica( File parentDir, int serverId, int boltPort, int httpPort, in config.put( CausalClusteringSettings.transaction_listen_address.name(), listenAddress( listenAddress, txPort ) ); config.put( OnlineBackupSettings.online_backup_server.name(), listenAddress( listenAddress, backupPort ) ); config.put( GraphDatabaseSettings.logs_directory.name(), new File( neo4jHome, "logs" ).getAbsolutePath() ); + config.put( GraphDatabaseSettings.logical_logs_location.name(), "replica-tx-logs-" + serverId ); this.discoveryServiceFactory = discoveryServiceFactory; storeDir = new File( new File( new File( neo4jHome, "data" ), "databases" ), "graph.db" ); + //noinspection ResultOfMethodCallIgnored storeDir.mkdirs(); diff --git a/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/scenarios/ClusterCustomLogLocationIT.java b/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/scenarios/ClusterCustomLogLocationIT.java new file mode 100644 index 0000000000000..d00c37769cbbc --- /dev/null +++ b/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/scenarios/ClusterCustomLogLocationIT.java @@ -0,0 +1,99 @@ +/* + * Copyright (c) 2002-2017 "Neo Technology," + * Network Engine for Objects in Lund AB [http://neotechnology.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +package org.neo4j.causalclustering.scenarios; + +import org.hamcrest.Matchers; +import org.junit.Rule; +import org.junit.Test; + +import java.io.File; +import java.io.IOException; +import java.util.Collection; + +import org.neo4j.causalclustering.discovery.Cluster; +import org.neo4j.causalclustering.discovery.CoreClusterMember; +import org.neo4j.causalclustering.discovery.ReadReplica; +import org.neo4j.graphdb.DependencyResolver; +import org.neo4j.io.fs.FileSystemAbstraction; +import org.neo4j.kernel.impl.transaction.log.files.LogFiles; +import org.neo4j.test.causalclustering.ClusterRule; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertTrue; +import static org.neo4j.kernel.impl.transaction.log.files.LogFilesBuilder.logFilesBasedOnlyBuilder; + +public class ClusterCustomLogLocationIT +{ + @Rule + public final ClusterRule clusterRule = new ClusterRule( getClass() ) + .withNumberOfCoreMembers( 3 ) + .withNumberOfReadReplicas( 2 ); + + @Test + public void clusterWithCustomTransactionLogLocation() throws Exception + { + Cluster cluster = clusterRule.startCluster(); + + for ( int i = 0; i < 10; i++ ) + { + cluster.coreTx( ( db, tx ) -> + { + db.createNode(); + tx.success(); + } ); + } + + Collection coreClusterMembers = cluster.coreMembers(); + for ( CoreClusterMember coreClusterMember : coreClusterMembers ) + { + DependencyResolver dependencyResolver = coreClusterMember.database().getDependencyResolver(); + LogFiles logFiles = dependencyResolver.resolveDependency( LogFiles.class ); + assertEquals( logFiles.logFilesDirectory().getName(), "core-tx-logs-" + coreClusterMember.serverId() ); + assertTrue( logFiles.hasAnyEntries( 0 ) ); + File[] coreLogDirectories = coreClusterMember.storeDir().listFiles( file -> file.getName().startsWith( "core" ) ); + assertThat( coreLogDirectories, Matchers.arrayWithSize( 1 ) ); + + logFileInStoreDirectoryDoesNotExist( coreClusterMember.storeDir(), dependencyResolver ); + } + + Collection readReplicas = cluster.readReplicas(); + for ( ReadReplica readReplica : readReplicas ) + { + readReplica.txPollingClient().upToDateFuture().get(); + DependencyResolver dependencyResolver = readReplica.database().getDependencyResolver(); + LogFiles logFiles = dependencyResolver.resolveDependency( LogFiles.class ); + assertEquals( logFiles.logFilesDirectory().getName(), "replica-tx-logs-" + readReplica.serverId() ); + assertTrue( logFiles.hasAnyEntries( 0 ) ); + File[] replicaLogDirectories = readReplica.storeDir().listFiles( file -> file.getName().startsWith( "replica" ) ); + assertThat( replicaLogDirectories, Matchers.arrayWithSize( 1 ) ); + + logFileInStoreDirectoryDoesNotExist( readReplica.storeDir(), dependencyResolver ); + } + } + + private void logFileInStoreDirectoryDoesNotExist( File storeDir, DependencyResolver dependencyResolver ) throws IOException + { + FileSystemAbstraction fileSystem = dependencyResolver.resolveDependency( FileSystemAbstraction.class ); + LogFiles storeLogFiles = logFilesBasedOnlyBuilder( storeDir, fileSystem ).build(); + assertFalse( storeLogFiles.versionExists( 0 ) ); + } +} diff --git a/enterprise/com/src/main/java/org/neo4j/com/storecopy/FileMoveAction.java b/enterprise/com/src/main/java/org/neo4j/com/storecopy/FileMoveAction.java index a0468235cfe95..ec606137e8a44 100644 --- a/enterprise/com/src/main/java/org/neo4j/com/storecopy/FileMoveAction.java +++ b/enterprise/com/src/main/java/org/neo4j/com/storecopy/FileMoveAction.java @@ -29,33 +29,55 @@ import org.neo4j.io.fs.FileHandle; import org.neo4j.io.pagecache.PageCache; -@FunctionalInterface public interface FileMoveAction { void move( File toDir, CopyOption... copyOptions ) throws IOException; + File file(); + static FileMoveAction copyViaPageCache( File file, PageCache pageCache ) { - return ( toDir, copyOptions ) -> + return new FileMoveAction() { - Optional handle = pageCache.getCachedFileSystem().streamFilesRecursive( file ).findAny(); - if ( handle.isPresent() ) + + @Override + public void move( File toDir, CopyOption... copyOptions ) throws IOException + { + Optional handle = pageCache.getCachedFileSystem().streamFilesRecursive( file ).findAny(); + if ( handle.isPresent() ) + { + handle.get().rename( new File( toDir, file.getName() ), copyOptions ); + } + } + + @Override + public File file() { - handle.get().rename( new File( toDir, file.getName() ), copyOptions ); + return file; } }; } static FileMoveAction copyViaFileSystem( File file, File basePath ) { - Path base = basePath.toPath(); - return ( toDir, copyOptions ) -> + return new FileMoveAction() { - Path originalPath = file.toPath(); - Path relativePath = base.relativize( originalPath ); - Path resolvedPath = toDir.toPath().resolve( relativePath ); - Files.createDirectories( resolvedPath.getParent() ); - Files.copy( originalPath, resolvedPath, copyOptions ); + @Override + public void move( File toDir, CopyOption... copyOptions ) throws IOException + { + Path base = basePath.toPath(); + Path originalPath = file.toPath(); + Path relativePath = base.relativize( originalPath ); + Path resolvedPath = toDir.toPath().resolve( relativePath ); + Files.createDirectories( resolvedPath.getParent() ); + Files.copy( originalPath, resolvedPath, copyOptions ); + } + + @Override + public File file() + { + return file; + } }; } diff --git a/enterprise/com/src/main/java/org/neo4j/com/storecopy/MoveAfterCopy.java b/enterprise/com/src/main/java/org/neo4j/com/storecopy/MoveAfterCopy.java index 1563c047e7b05..7e37e493077a6 100644 --- a/enterprise/com/src/main/java/org/neo4j/com/storecopy/MoveAfterCopy.java +++ b/enterprise/com/src/main/java/org/neo4j/com/storecopy/MoveAfterCopy.java @@ -21,20 +21,23 @@ import java.io.File; import java.nio.file.StandardCopyOption; +import java.util.function.Function; import java.util.stream.Stream; +@FunctionalInterface public interface MoveAfterCopy { - void move( Stream moves, File fromDirectory, File toDirectory ) throws Exception; + void move( Stream moves, File fromDirectory, Function destinationFunction ) throws + Exception; static MoveAfterCopy moveReplaceExisting() { - return ( moves, fromDirectory, toDirectory ) -> + return ( moves, fromDirectory, destinationFunction ) -> { Iterable itr = moves::iterator; for ( FileMoveAction move : itr ) { - move.move( toDirectory, StandardCopyOption.REPLACE_EXISTING ); + move.move( destinationFunction.apply( move.file() ), StandardCopyOption.REPLACE_EXISTING ); } }; } diff --git a/enterprise/com/src/main/java/org/neo4j/com/storecopy/StoreCopyClient.java b/enterprise/com/src/main/java/org/neo4j/com/storecopy/StoreCopyClient.java index 3c2e2832969ed..72d549eb4880d 100644 --- a/enterprise/com/src/main/java/org/neo4j/com/storecopy/StoreCopyClient.java +++ b/enterprise/com/src/main/java/org/neo4j/com/storecopy/StoreCopyClient.java @@ -27,6 +27,7 @@ import java.util.Arrays; import java.util.List; import java.util.concurrent.atomic.AtomicLong; +import java.util.function.Function; import java.util.stream.Stream; import org.neo4j.com.Response; @@ -203,13 +204,18 @@ public void copyStore( StoreCopyRequester requester, CancellationRequest cancell graphDatabaseService.shutdown(); monitor.finishRecoveringStore(); + LogFiles logFiles = LogFilesBuilder.activeFilesBuilder( storeDir, fs, pageCache ) + .withConfig( config ) + .build(); // All is well, move the streamed files to the real store directory. // Start with the files written through the page cache. Should only be record store files. // Note that the stream is lazy, so the file system traversal won't happen until *after* the store files // have been moved. Thus we ensure that we only attempt to move them once. Stream moveActionStream = Stream.concat( storeFileMoveActions.stream(), traverseGenerateMoveActions( tempStore, tempStore ) ); - moveAfterCopy.move( moveActionStream, tempStore, storeDir ); + Function destinationMapper = + file -> logFiles.isLogFile( file ) ? logFiles.logFilesDirectory() : storeDir; + moveAfterCopy.move( moveActionStream, tempStore, destinationMapper ); } finally { @@ -333,6 +339,7 @@ private GraphDatabaseService newTempDatabase( File tempStore ) .setConfig( "dbms.backup.enabled", Settings.FALSE ) .setConfig( GraphDatabaseSettings.logs_directory, tempStore.getAbsolutePath() ) .setConfig( GraphDatabaseSettings.keep_logical_logs, Settings.TRUE ) + .setConfig( GraphDatabaseSettings.logical_logs_location, tempStore.getAbsolutePath() ) .setConfig( GraphDatabaseSettings.allow_upgrade, config.get( GraphDatabaseSettings.allow_upgrade ).toString() ) .newGraphDatabase(); diff --git a/enterprise/com/src/main/java/org/neo4j/com/storecopy/StoreCopyServer.java b/enterprise/com/src/main/java/org/neo4j/com/storecopy/StoreCopyServer.java index fbe03664cd80d..00dc64c9292dc 100644 --- a/enterprise/com/src/main/java/org/neo4j/com/storecopy/StoreCopyServer.java +++ b/enterprise/com/src/main/java/org/neo4j/com/storecopy/StoreCopyServer.java @@ -174,6 +174,7 @@ public RequestContext flushStoresAndStreamStoreFiles( String triggerName, StoreW { StoreFileMetadata meta = files.next(); File file = meta.file(); + boolean isLogFile = meta.isLogFile(); int recordSize = meta.recordSize(); // Read from paged file if mapping exists. Otherwise read through file system. @@ -187,7 +188,8 @@ public RequestContext flushStoresAndStreamStoreFiles( String triggerName, StoreW long fileSize = pagedFile.fileSize(); try ( ReadableByteChannel fileChannel = pagedFile.openReadableByteChannel() ) { - doWrite( writer, temporaryBuffer, file, recordSize, fileChannel, fileSize, storeCopyIdentifier ); + doWrite( writer, temporaryBuffer, file, recordSize, fileChannel, fileSize, + storeCopyIdentifier, false ); } } } @@ -196,7 +198,8 @@ public RequestContext flushStoresAndStreamStoreFiles( String triggerName, StoreW try ( ReadableByteChannel fileChannel = fileSystem.open( file, OpenMode.READ ) ) { long fileSize = fileSystem.getFileSize( file ); - doWrite( writer, temporaryBuffer, file, recordSize, fileChannel, fileSize, storeCopyIdentifier ); + doWrite( writer, temporaryBuffer, file, recordSize, fileChannel, fileSize, + storeCopyIdentifier, isLogFile ); } } } @@ -215,11 +218,11 @@ public RequestContext flushStoresAndStreamStoreFiles( String triggerName, StoreW } private void doWrite( StoreWriter writer, ByteBuffer temporaryBuffer, File file, int recordSize, - ReadableByteChannel fileChannel, long fileSize, String storeCopyIdentifier ) throws IOException + ReadableByteChannel fileChannel, long fileSize, String storeCopyIdentifier, boolean isLogFile ) throws IOException { monitor.startStreamingStoreFile( file, storeCopyIdentifier ); - writer.write( relativePath( storeDirectory, file ), fileChannel, - temporaryBuffer, fileSize > 0, recordSize ); + String path = isLogFile ? file.getName() : relativePath( storeDirectory, file ); + writer.write( path, fileChannel, temporaryBuffer, fileSize > 0, recordSize ); monitor.finishStreamingStoreFile( file, storeCopyIdentifier ); } } diff --git a/enterprise/com/src/test/java/org/neo4j/com/storecopy/StoreCopyClientTest.java b/enterprise/com/src/test/java/org/neo4j/com/storecopy/StoreCopyClientTest.java index b51322101ddc3..6e8d7c62621d5 100644 --- a/enterprise/com/src/test/java/org/neo4j/com/storecopy/StoreCopyClientTest.java +++ b/enterprise/com/src/test/java/org/neo4j/com/storecopy/StoreCopyClientTest.java @@ -52,6 +52,8 @@ import org.neo4j.kernel.impl.transaction.log.checkpoint.CheckPointer; import org.neo4j.kernel.impl.transaction.log.checkpoint.StoreCopyCheckPointMutex; import org.neo4j.kernel.impl.transaction.log.entry.LogHeader; +import org.neo4j.kernel.impl.transaction.log.files.LogFiles; +import org.neo4j.kernel.impl.transaction.log.files.LogFilesBuilder; import org.neo4j.kernel.internal.GraphDatabaseAPI; import org.neo4j.kernel.monitoring.Monitors; import org.neo4j.logging.NullLogProvider; @@ -62,28 +64,31 @@ import org.neo4j.test.rule.fs.DefaultFileSystemRule; import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.greaterThanOrEqualTo; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotEquals; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; import static org.mockito.Mockito.spy; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.neo4j.com.storecopy.StoreUtil.TEMP_COPY_DIRECTORY_NAME; import static org.neo4j.graphdb.Label.label; +import static org.neo4j.graphdb.factory.GraphDatabaseSettings.logical_logs_location; import static org.neo4j.graphdb.factory.GraphDatabaseSettings.record_format; public class StoreCopyClientTest { - private final TestDirectory testDir = TestDirectory.testDirectory(); + private final TestDirectory directory = TestDirectory.testDirectory(); private final PageCacheRule pageCacheRule = new PageCacheRule(); private final CleanupRule cleanup = new CleanupRule(); private final DefaultFileSystemRule fileSystemRule = new DefaultFileSystemRule(); @Rule - public TestRule rules = RuleChain.outerRule( testDir ).around( fileSystemRule ). + public TestRule rules = RuleChain.outerRule( directory ).around( fileSystemRule ). around( pageCacheRule ).around( cleanup ); private FileSystemAbstraction fileSystem; @@ -99,8 +104,8 @@ public void shouldCopyStoreFilesAcrossIfACancellationRequestHappensAfterTheTempS throws Exception { // given - final File copyDir = new File( testDir.directory(), "copy" ); - final File originalDir = new File( testDir.directory(), "original" ); + final File copyDir = new File( directory.directory(), "copy" ); + final File originalDir = new File( directory.directory(), "original" ); final AtomicBoolean cancelStoreCopy = new AtomicBoolean( false ); StoreCopyClient.Monitor storeCopyMonitor = new StoreCopyClient.Monitor.Adapter() @@ -129,7 +134,7 @@ public void finishRecoveringStore() } StoreCopyClient.StoreCopyRequester storeCopyRequest = - spy( new LocalStoreCopyRequester( original, originalDir, fileSystem ) ); + spy( new LocalStoreCopyRequester( original, originalDir, fileSystem, false ) ); // when copier.copyStore( storeCopyRequest, cancelStoreCopy::get, MoveAfterCopy.moveReplaceExisting() ); @@ -153,37 +158,58 @@ public void finishRecoveringStore() } @Test - public void storeCopyClientMustWorkWithStandardRecordFormat() throws Exception - { - checkStoreCopyClientWithRecordFormats( Standard.LATEST_NAME ); - } - - @Test - public void storeCopyClientMustWorkWithHighLimitRecordFormat() throws Exception + public void storeCopyClientUseCustomTransactionLogLocationWhenConfigured() throws Exception { - checkStoreCopyClientWithRecordFormats( HighLimit.NAME ); - } - - private void checkStoreCopyClientWithRecordFormats( String recordFormatsName ) throws Exception - { - final File copyDir = new File( testDir.directory(), "copy" ); - final File originalDir = new File( testDir.directory(), "original" ); + final File copyDir = new File( directory.directory(), "copyCustomLocation" ); + final File originalDir = new File( directory.directory(), "originalCustomLocation" ); PageCache pageCache = pageCacheRule.getPageCache( fileSystem ); - Config config = Config.defaults( record_format, recordFormatsName ); + File copyCustomLogFilesLocation = new File( copyDir, "CopyCustomLogFilesLocation" ); + File originalCustomLogFilesLocation = new File( originalDir, "originalCustomLogFilesLocation" ); + + Config config = Config.defaults( logical_logs_location, copyCustomLogFilesLocation.getName() ); StoreCopyClient copier = new StoreCopyClient( copyDir, config, loadKernelExtensions(), NullLogProvider.getInstance(), fileSystem, pageCache, new StoreCopyClient.Monitor.Adapter(), false ); - final GraphDatabaseAPI original = (GraphDatabaseAPI) startDatabase( originalDir, recordFormatsName ); + GraphDatabaseAPI original = (GraphDatabaseAPI) new TestGraphDatabaseFactory() + .newEmbeddedDatabaseBuilder( originalDir ) + .setConfig( logical_logs_location, originalCustomLogFilesLocation.getName() ) + .newGraphDatabase(); + generateTransactions( original ); + long logFileSize = + original.getDependencyResolver().resolveDependency( LogFiles.class ).getLogFileForVersion( 0 ).length(); + StoreCopyClient.StoreCopyRequester storeCopyRequest = new LocalStoreCopyRequester( original, originalDir, - fileSystem ); + fileSystem, true ); copier.copyStore( storeCopyRequest, CancellationRequest.NEVER_CANCELLED, MoveAfterCopy.moveReplaceExisting() ); + original.shutdown(); assertFalse( new File( copyDir, TEMP_COPY_DIRECTORY_NAME ).exists() ); - // Must not throw - startDatabase( copyDir, recordFormatsName ).shutdown(); + LogFiles customLogFiles = LogFilesBuilder.logFilesBasedOnlyBuilder( copyCustomLogFilesLocation, fileSystem ).build(); + assertTrue( customLogFiles.versionExists( 0 ) ); + assertThat( customLogFiles.getLogFileForVersion( 0 ).length(), greaterThanOrEqualTo( logFileSize ) ); + + LogFiles logFiles = LogFilesBuilder.logFilesBasedOnlyBuilder( copyDir, fileSystem ).build(); + assertFalse( logFiles.versionExists( 0 ) ); + + new TestGraphDatabaseFactory() + .newEmbeddedDatabaseBuilder( copyDir ) + .setConfig( logical_logs_location, copyCustomLogFilesLocation.getName() ) + .newGraphDatabase().shutdown(); + } + + @Test + public void storeCopyClientMustWorkWithStandardRecordFormat() throws Exception + { + checkStoreCopyClientWithRecordFormats( Standard.LATEST_NAME ); + } + + @Test + public void storeCopyClientMustWorkWithHighLimitRecordFormat() throws Exception + { + checkStoreCopyClientWithRecordFormats( HighLimit.NAME ); } @Test @@ -191,8 +217,8 @@ public void shouldEndUpWithAnEmptyStoreIfCancellationRequestIssuedJustBeforeReco throws Exception { // given - final File copyDir = new File( testDir.directory(), "copy" ); - final File originalDir = new File( testDir.directory(), "original" ); + final File copyDir = new File( directory.directory(), "copy" ); + final File originalDir = new File( directory.directory(), "original" ); final AtomicBoolean cancelStoreCopy = new AtomicBoolean( false ); StoreCopyClient.Monitor storeCopyMonitor = new StoreCopyClient.Monitor.Adapter() @@ -219,7 +245,7 @@ public void finishReceivingStoreFiles() } StoreCopyClient.StoreCopyRequester storeCopyRequest = - spy( new LocalStoreCopyRequester( original, originalDir, fileSystem ) ); + spy( new LocalStoreCopyRequester( original, originalDir, fileSystem, false ) ); // when copier.copyStore( storeCopyRequest, cancelStoreCopy::get, MoveAfterCopy.moveReplaceExisting() ); @@ -243,8 +269,8 @@ public void finishReceivingStoreFiles() public void shouldResetNeoStoreLastTransactionOffsetForNonForensicCopy() throws Exception { // GIVEN - File initialStore = testDir.directory( "initialStore" ); - File backupStore = testDir.directory( "backupStore" ); + File initialStore = directory.directory( "initialStore" ); + File backupStore = directory.directory( "backupStore" ); PageCache pageCache = pageCacheRule.getPageCache( fileSystem ); createInitialDatabase( initialStore ); @@ -259,7 +285,7 @@ public void shouldResetNeoStoreLastTransactionOffsetForNonForensicCopy() throws .getInstance(), fileSystem, pageCache, new StoreCopyClient.Monitor.Adapter(), false ); CancellationRequest falseCancellationRequest = () -> false; StoreCopyClient.StoreCopyRequester storeCopyRequest = - new LocalStoreCopyRequester( (GraphDatabaseAPI) initialDatabase, initialStore, fileSystem ); + new LocalStoreCopyRequester( (GraphDatabaseAPI) initialDatabase, initialStore, fileSystem, false ); // WHEN copier.copyStore( storeCopyRequest, falseCancellationRequest, MoveAfterCopy.moveReplaceExisting() ); @@ -277,8 +303,8 @@ public void shouldResetNeoStoreLastTransactionOffsetForNonForensicCopy() throws public void shouldDeleteTempCopyFolderOnFailures() throws Exception { // GIVEN - File initialStore = testDir.directory( "initialStore" ); - File backupStore = testDir.directory( "backupStore" ); + File initialStore = directory.directory( "initialStore" ); + File backupStore = directory.directory( "backupStore" ); PageCache pageCache = pageCacheRule.getPageCache( fileSystem ); GraphDatabaseService initialDatabase = createInitialDatabase( initialStore ); @@ -289,7 +315,7 @@ public void shouldDeleteTempCopyFolderOnFailures() throws Exception RuntimeException exception = new RuntimeException( "Boom!" ); StoreCopyClient.StoreCopyRequester storeCopyRequest = - new LocalStoreCopyRequester( (GraphDatabaseAPI) initialDatabase, initialStore, fileSystem ) + new LocalStoreCopyRequester( (GraphDatabaseAPI) initialDatabase, initialStore, fileSystem, false ) { @Override public void done() @@ -313,6 +339,28 @@ public void done() assertFalse( new File( backupStore, TEMP_COPY_DIRECTORY_NAME ).exists() ); } + private void checkStoreCopyClientWithRecordFormats( String recordFormatsName ) throws Exception + { + final File copyDir = new File( directory.directory(), "copy" ); + final File originalDir = new File( directory.directory(), "original" ); + PageCache pageCache = pageCacheRule.getPageCache( fileSystem ); + Config config = Config.defaults( record_format, recordFormatsName ); + StoreCopyClient copier = new StoreCopyClient( + copyDir, config, loadKernelExtensions(), NullLogProvider.getInstance(), fileSystem, pageCache, + new StoreCopyClient.Monitor.Adapter(), false ); + + final GraphDatabaseAPI original = (GraphDatabaseAPI) startDatabase( originalDir, recordFormatsName ); + StoreCopyClient.StoreCopyRequester storeCopyRequest = new LocalStoreCopyRequester( original, originalDir, + fileSystem, false ); + + copier.copyStore( storeCopyRequest, CancellationRequest.NEVER_CANCELLED, MoveAfterCopy.moveReplaceExisting() ); + + assertFalse( new File( copyDir, TEMP_COPY_DIRECTORY_NAME ).exists() ); + + // Must not throw + startDatabase( copyDir, recordFormatsName ).shutdown(); + } + private GraphDatabaseService createInitialDatabase( File initialStore ) { GraphDatabaseService initialDatabase = startDatabase( initialStore ); @@ -352,6 +400,18 @@ private static List> loadKernelExtensions() return kernelExtensions; } + private void generateTransactions( GraphDatabaseAPI original ) + { + for ( int i = 0; i < 10; i++ ) + { + try ( Transaction transaction = original.beginTx() ) + { + original.createNode(); + transaction.success(); + } + } + } + private static class LocalStoreCopyRequester implements StoreCopyClient.StoreCopyRequester { private final GraphDatabaseAPI original; @@ -359,12 +419,15 @@ private static class LocalStoreCopyRequester implements StoreCopyClient.StoreCop private final FileSystemAbstraction fs; private Response response; + private boolean includeLogs; - LocalStoreCopyRequester( GraphDatabaseAPI original, File originalDir, FileSystemAbstraction fs ) + LocalStoreCopyRequester( GraphDatabaseAPI original, File originalDir, FileSystemAbstraction fs, + boolean includeLogs ) { this.original = original; this.originalDir = originalDir; this.fs = fs; + this.includeLogs = includeLogs; } @Override @@ -388,7 +451,7 @@ public Response copyStore( StoreWriter writer ) RequestContext requestContext = new StoreCopyServer( neoStoreDataSource, checkPointer, fs, originalDir, new Monitors().newMonitor( StoreCopyServer.Monitor.class ), pageCache, new StoreCopyCheckPointMutex() ) - .flushStoresAndStreamStoreFiles( "test", writer, false ); + .flushStoresAndStreamStoreFiles( "test", writer, includeLogs ); final StoreId storeId = original.getDependencyResolver().resolveDependency( RecordStorageEngine.class ) diff --git a/enterprise/fulltext-addon/src/test/java/org/neo4j/kernel/api/impl/fulltext/integrations/bloom/BloomIT.java b/enterprise/fulltext-addon/src/test/java/org/neo4j/kernel/api/impl/fulltext/integrations/bloom/BloomIT.java index 2c272851cdd52..100bb484c4fc6 100644 --- a/enterprise/fulltext-addon/src/test/java/org/neo4j/kernel/api/impl/fulltext/integrations/bloom/BloomIT.java +++ b/enterprise/fulltext-addon/src/test/java/org/neo4j/kernel/api/impl/fulltext/integrations/bloom/BloomIT.java @@ -19,7 +19,6 @@ */ package org.neo4j.kernel.api.impl.fulltext.integrations.bloom; -import org.apache.commons.lang3.SystemUtils; import org.apache.lucene.analysis.en.EnglishAnalyzer; import org.apache.lucene.analysis.sv.SwedishAnalyzer; import org.junit.After; @@ -28,7 +27,6 @@ import org.junit.Test; import org.junit.rules.ExpectedException; -import java.io.File; import java.util.Date; import org.neo4j.consistency.ConsistencyCheckService; @@ -57,8 +55,6 @@ import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertThat; import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; -import static org.junit.Assume.assumeFalse; import static org.neo4j.kernel.api.impl.fulltext.integrations.bloom.BloomFulltextConfig.bloom_enabled; public class BloomIT @@ -89,8 +85,8 @@ public class BloomIT public void before() throws Exception { GraphDatabaseFactory factory = new GraphDatabaseFactory(); - builder = factory.newEmbeddedDatabaseBuilder( testDirectory.graphDbDir() ); - builder.setConfig( bloom_enabled, "true" ); + builder = factory.newEmbeddedDatabaseBuilder( testDirectory.graphDbDir() ) + .setConfig( bloom_enabled, "true" ); } @After @@ -604,58 +600,4 @@ public void onlineIndexShouldBeReportedAsOnline() throws Exception assertEquals( "ONLINE", result.next().get( "state" ) ); assertFalse( result.hasNext() ); } - - @Test - public void failureToStartUpMustNotPreventShutDown() throws Exception - { - // Ignore this test on Windows because the test relies on file permissions to trigger failure modes in - // the code. Unfortunately, file permissions are an incredible pain to work with on Windows. - assumeFalse( SystemUtils.IS_OS_WINDOWS ); - - // Create the store directory and all its files, and add a bit of data to it - GraphDatabaseService db = getDb(); - db.execute( String.format( SET_NODE_KEYS, "\"prop\"" ) ); - - try ( Transaction tx = db.beginTx() ) - { - db.createNode().setProperty( "prop", "bla bla bla" ); - tx.success(); - } - db.shutdown(); - - File dir = testDirectory.graphDbDir(); - assertTrue( dir.setReadable( false ) ); - try - { - // Making the directory not readable ought to cause problems for the database as it tries to start up - getDb().shutdown(); - fail( "Should not have started up and shut down cleanly on an unreadable store directory" ); - } - catch ( Exception e ) - { - // Good - } - catch ( Throwable th ) - { - makeReadable( dir, th ); - throw th; - } - makeReadable( dir, null ); - } - - private void makeReadable( File dir, Throwable th ) - { - if ( !dir.setReadable( true ) ) - { - AssertionError error = new AssertionError( "Failed to make " + dir + " writable again!" ); - if ( th != null ) - { - th.addSuppressed( error ); - } - else - { - throw error; - } - } } -} diff --git a/enterprise/ha/src/test/java/org/neo4j/kernel/ha/cluster/SwitchToSlaveCopyThenBranchTest.java b/enterprise/ha/src/test/java/org/neo4j/kernel/ha/cluster/SwitchToSlaveCopyThenBranchTest.java index 1717f3d7b591a..8d5667e1da732 100644 --- a/enterprise/ha/src/test/java/org/neo4j/kernel/ha/cluster/SwitchToSlaveCopyThenBranchTest.java +++ b/enterprise/ha/src/test/java/org/neo4j/kernel/ha/cluster/SwitchToSlaveCopyThenBranchTest.java @@ -29,6 +29,7 @@ import java.net.URI; import java.net.URISyntaxException; import java.util.concurrent.TimeUnit; +import java.util.function.Function; import java.util.function.Predicate; import java.util.stream.Stream; @@ -231,7 +232,7 @@ public void shouldNotBranchStoreUnlessWeHaveCopiedDownAReplacement() throws Thro doAnswer( invocation -> { MoveAfterCopy moveAfterCopy = invocation.getArgument( 2 ); - moveAfterCopy.move( Stream.empty(), new File( "" ), new File( "" ) ); + moveAfterCopy.move( Stream.empty(), new File( "" ), Function.identity() ); return null; } ).when( storeCopyClient ).copyStore( any( StoreCopyClient.StoreCopyRequester.class ), diff --git a/enterprise/kernel/src/test/java/org/neo4j/util/TestHelpers.java b/enterprise/kernel/src/test/java/org/neo4j/util/TestHelpers.java index 18705a3143dcd..3593ce665f16b 100644 --- a/enterprise/kernel/src/test/java/org/neo4j/util/TestHelpers.java +++ b/enterprise/kernel/src/test/java/org/neo4j/util/TestHelpers.java @@ -25,22 +25,16 @@ import java.util.List; import org.neo4j.commandline.admin.AdminTool; -import org.neo4j.helpers.AdvertisedSocketAddress; import org.neo4j.helpers.HostnamePort; import org.neo4j.helpers.ListenSocketAddress; import org.neo4j.io.proc.ProcessUtil; import org.neo4j.kernel.configuration.Config; -import org.neo4j.kernel.configuration.Settings; import org.neo4j.kernel.impl.enterprise.configuration.OnlineBackupSettings; -import org.neo4j.kernel.impl.factory.GraphDatabaseFacade; import org.neo4j.kernel.internal.GraphDatabaseAPI; import org.neo4j.test.ProcessStreamHandler; -import org.neo4j.test.rule.DatabaseRule; -import org.neo4j.test.rule.EmbeddedDatabaseRule; import static java.lang.String.format; import static org.neo4j.kernel.configuration.Settings.listenAddress; -import static org.neo4j.kernel.configuration.Settings.setting; public class TestHelpers { diff --git a/enterprise/server-enterprise/src/main/java/org/neo4j/server/enterprise/EnterpriseNeoServer.java b/enterprise/server-enterprise/src/main/java/org/neo4j/server/enterprise/EnterpriseNeoServer.java index 58b60b9aa5a6d..51442ac6646b3 100644 --- a/enterprise/server-enterprise/src/main/java/org/neo4j/server/enterprise/EnterpriseNeoServer.java +++ b/enterprise/server-enterprise/src/main/java/org/neo4j/server/enterprise/EnterpriseNeoServer.java @@ -30,15 +30,15 @@ import org.neo4j.causalclustering.core.CausalClusteringSettings; import org.neo4j.causalclustering.core.CoreGraphDatabase; import org.neo4j.causalclustering.readreplica.ReadReplicaGraphDatabase; -import org.neo4j.dbms.DatabaseManagementSystemSettings; +import org.neo4j.graphdb.factory.GraphDatabaseSettings; import org.neo4j.helpers.collection.Iterables; import org.neo4j.kernel.configuration.Config; import org.neo4j.kernel.enterprise.EnterpriseGraphDatabase; import org.neo4j.kernel.ha.HaSettings; import org.neo4j.kernel.ha.HighlyAvailableGraphDatabase; -import org.neo4j.kernel.impl.factory.GraphDatabaseFacadeFactory; import org.neo4j.kernel.impl.enterprise.configuration.EnterpriseEditionSettings; import org.neo4j.kernel.impl.enterprise.configuration.EnterpriseEditionSettings.Mode; +import org.neo4j.kernel.impl.factory.GraphDatabaseFacadeFactory; import org.neo4j.kernel.impl.factory.GraphDatabaseFacadeFactory.Dependencies; import org.neo4j.kernel.impl.util.UnsatisfiedDependencyException; import org.neo4j.logging.LogProvider; @@ -65,25 +65,25 @@ public class EnterpriseNeoServer extends CommunityNeoServer private static final GraphFactory HA_FACTORY = ( config, dependencies ) -> { - File storeDir = config.get( DatabaseManagementSystemSettings.database_path ); + File storeDir = config.get( GraphDatabaseSettings.database_path ); return new HighlyAvailableGraphDatabase( storeDir, config, dependencies ); }; private static final GraphFactory ENTERPRISE_FACTORY = ( config, dependencies ) -> { - File storeDir = config.get( DatabaseManagementSystemSettings.database_path ); + File storeDir = config.get( GraphDatabaseSettings.database_path ); return new EnterpriseGraphDatabase( storeDir, config, dependencies ); }; private static final GraphFactory CORE_FACTORY = ( config, dependencies ) -> { - File storeDir = config.get( DatabaseManagementSystemSettings.database_path ); + File storeDir = config.get( GraphDatabaseSettings.database_path ); return new CoreGraphDatabase( storeDir, config, dependencies ); }; private static final GraphFactory READ_REPLICA_FACTORY = ( config, dependencies ) -> { - File storeDir = config.get( DatabaseManagementSystemSettings.database_path ); + File storeDir = config.get( GraphDatabaseSettings.database_path ); return new ReadReplicaGraphDatabase( storeDir, config, dependencies ); }; diff --git a/enterprise/server-enterprise/src/test/java/org/neo4j/server/enterprise/EnterpriseBootstrapperTestIT.java b/enterprise/server-enterprise/src/test/java/org/neo4j/server/enterprise/EnterpriseBootstrapperTestIT.java index 0fbd2241d1911..9053deed07e87 100644 --- a/enterprise/server-enterprise/src/test/java/org/neo4j/server/enterprise/EnterpriseBootstrapperTestIT.java +++ b/enterprise/server-enterprise/src/test/java/org/neo4j/server/enterprise/EnterpriseBootstrapperTestIT.java @@ -45,7 +45,7 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; -import static org.neo4j.dbms.DatabaseManagementSystemSettings.data_directory; +import static org.neo4j.graphdb.factory.GraphDatabaseSettings.data_directory; import static org.neo4j.graphdb.factory.GraphDatabaseSettings.logs_directory; import static org.neo4j.graphdb.factory.GraphDatabaseSettings.store_internal_log_level; import static org.neo4j.helpers.collection.MapUtil.store; diff --git a/enterprise/server-enterprise/src/test/java/org/neo4j/server/enterprise/jmx/ServerManagementIT.java b/enterprise/server-enterprise/src/test/java/org/neo4j/server/enterprise/jmx/ServerManagementIT.java index 5c47836547e62..fe71fb52dade1 100644 --- a/enterprise/server-enterprise/src/test/java/org/neo4j/server/enterprise/jmx/ServerManagementIT.java +++ b/enterprise/server-enterprise/src/test/java/org/neo4j/server/enterprise/jmx/ServerManagementIT.java @@ -23,7 +23,6 @@ import org.junit.Test; import org.junit.rules.RuleChain; -import org.neo4j.dbms.DatabaseManagementSystemSettings; import org.neo4j.graphdb.factory.GraphDatabaseSettings; import org.neo4j.kernel.GraphDatabaseDependencies; import org.neo4j.kernel.configuration.Config; @@ -70,17 +69,17 @@ public void shouldBeAbleToRestartServer() throws Exception server.start(); assertNotNull( server.getDatabase().getGraph() ); - assertEquals( config.get( DatabaseManagementSystemSettings.database_path ).getAbsolutePath(), + assertEquals( config.get( GraphDatabaseSettings.database_path ).getAbsolutePath(), server.getDatabase().getLocation().getAbsolutePath() ); // Change the database location - config.augment( DatabaseManagementSystemSettings.data_directory, dataDirectory2 ); + config.augment( GraphDatabaseSettings.data_directory, dataDirectory2 ); ServerManagement bean = new ServerManagement( server ); bean.restartServer(); // Then assertNotNull( server.getDatabase().getGraph() ); - assertEquals( config.get( DatabaseManagementSystemSettings.database_path ).getAbsolutePath(), + assertEquals( config.get( GraphDatabaseSettings.database_path ).getAbsolutePath(), server.getDatabase().getLocation().getAbsolutePath() ); } diff --git a/integrationtests/src/test/java/org/neo4j/storeupgrade/StoreUpgradeIT.java b/integrationtests/src/test/java/org/neo4j/storeupgrade/StoreUpgradeIT.java index 68f5bb8b34475..4b16e3ba21f6b 100644 --- a/integrationtests/src/test/java/org/neo4j/storeupgrade/StoreUpgradeIT.java +++ b/integrationtests/src/test/java/org/neo4j/storeupgrade/StoreUpgradeIT.java @@ -39,7 +39,6 @@ import java.util.Properties; import org.neo4j.backup.OnlineBackupSettings; -import org.neo4j.dbms.DatabaseManagementSystemSettings; import org.neo4j.graphdb.GraphDatabaseService; import org.neo4j.graphdb.Label; import org.neo4j.graphdb.Node; @@ -49,12 +48,12 @@ import org.neo4j.graphdb.factory.GraphDatabaseSettings; import org.neo4j.helpers.Exceptions; import org.neo4j.helpers.collection.Iterables; +import org.neo4j.internal.kernel.api.exceptions.KernelException; import org.neo4j.io.fs.FileUtils; import org.neo4j.kernel.api.InwardKernel; import org.neo4j.kernel.api.KernelTransaction; import org.neo4j.kernel.api.ReadOperations; import org.neo4j.kernel.api.Statement; -import org.neo4j.internal.kernel.api.exceptions.KernelException; import org.neo4j.kernel.api.schema.index.IndexDescriptor; import org.neo4j.kernel.api.security.AnonymousContext; import org.neo4j.kernel.configuration.BoltConnector; @@ -171,15 +170,15 @@ public void embeddedDatabaseShouldStartOnOlderStoreWhenUpgradeIsEnabled() throws public void serverDatabaseShouldStartOnOlderStoreWhenUpgradeIsEnabled() throws Throwable { File rootDir = testDir.directory(); - File storeDir = Config.defaults( DatabaseManagementSystemSettings.data_directory, rootDir.toString() ) - .get( DatabaseManagementSystemSettings.database_path ); + File storeDir = Config.defaults( GraphDatabaseSettings.data_directory, rootDir.toString() ) + .get( GraphDatabaseSettings.database_path ); store.prepareDirectory( storeDir ); File configFile = new File( rootDir, Config.DEFAULT_CONFIG_FILE_NAME ); Properties props = new Properties(); props.putAll( ServerTestUtils.getDefaultRelativeProperties() ); - props.setProperty( DatabaseManagementSystemSettings.data_directory.name(), rootDir.getAbsolutePath() ); + props.setProperty( GraphDatabaseSettings.data_directory.name(), rootDir.getAbsolutePath() ); props.setProperty( GraphDatabaseSettings.logs_directory.name(), rootDir.getAbsolutePath() ); props.setProperty( GraphDatabaseSettings.allow_upgrade.name(), "true" ); props.setProperty( GraphDatabaseSettings.pagecache_memory.name(), "8m" ); diff --git a/tools/src/main/java/org/neo4j/tools/applytx/ApplyTransactionsCommand.java b/tools/src/main/java/org/neo4j/tools/applytx/ApplyTransactionsCommand.java index 06bb7ad984ad2..b1732d3837347 100644 --- a/tools/src/main/java/org/neo4j/tools/applytx/ApplyTransactionsCommand.java +++ b/tools/src/main/java/org/neo4j/tools/applytx/ApplyTransactionsCommand.java @@ -33,6 +33,7 @@ import org.neo4j.io.pagecache.PageCache; import org.neo4j.io.pagecache.impl.muninn.StandalonePageCacheFactory; import org.neo4j.kernel.api.exceptions.TransactionFailureException; +import org.neo4j.kernel.configuration.Config; import org.neo4j.kernel.impl.api.TransactionRepresentationCommitProcess; import org.neo4j.kernel.impl.api.TransactionToApply; import org.neo4j.kernel.impl.transaction.CommittedTransactionRepresentation; @@ -66,8 +67,9 @@ public ApplyTransactionsCommand( File from, Supplier to ) @Override protected void run( Args args, PrintStream out ) throws Exception { - TransactionIdStore txIdStore = to.get().getDependencyResolver().resolveDependency( - TransactionIdStore.class ); + DependencyResolver dependencyResolver = to.get().getDependencyResolver(); + TransactionIdStore txIdStore = dependencyResolver.resolveDependency( TransactionIdStore.class ); + Config config = dependencyResolver.resolveDependency( Config.class ); long fromTx = txIdStore.getLastCommittedTransaction().transactionId(); long toTx; if ( args.orphans().isEmpty() ) @@ -89,12 +91,12 @@ else if ( whereTo.equals( "last" ) ) toTx = Long.parseLong( whereTo ); } - long lastApplied = applyTransactions( from, to.get(), fromTx, toTx, out ); + long lastApplied = applyTransactions( from, to.get(), config, fromTx, toTx, out ); out.println( "Applied transactions up to and including " + lastApplied ); } - private long applyTransactions( File fromPath, GraphDatabaseAPI toDb, long fromTxExclusive, long toTxInclusive, - PrintStream out ) + private long applyTransactions( File fromPath, GraphDatabaseAPI toDb, Config toConfig, + long fromTxExclusive, long toTxInclusive, PrintStream out ) throws IOException, TransactionFailureException { DependencyResolver resolver = toDb.getDependencyResolver(); @@ -107,7 +109,7 @@ private long applyTransactions( File fromPath, GraphDatabaseAPI toDb, long fromT PageCache pageCache = StandalonePageCacheFactory.createPageCache( fileSystem ) ) { LogicalTransactionStore source = life.add( new ReadOnlyTransactionStore( pageCache, fileSystem, fromPath, - new Monitors() ) ); + Config.defaults(), new Monitors() ) ); life.start(); long lastAppliedTx = fromTxExclusive; // Some progress if there are more than a couple of transactions to apply diff --git a/tools/src/main/java/org/neo4j/tools/migration/StoreMigration.java b/tools/src/main/java/org/neo4j/tools/migration/StoreMigration.java index 7a982b4d47dd4..bdd142f33eef0 100644 --- a/tools/src/main/java/org/neo4j/tools/migration/StoreMigration.java +++ b/tools/src/main/java/org/neo4j/tools/migration/StoreMigration.java @@ -131,7 +131,8 @@ public void run( final FileSystemAbstraction fs, final File storeDirectory, Conf kernelContext, GraphDatabaseDependencies.newDependencies().kernelExtensions(), deps, ignore() ) ); - final LogFiles logFiles = LogFilesBuilder.activeFilesBuilder( storeDirectory, fs, pageCache ).build(); + final LogFiles logFiles = LogFilesBuilder.activeFilesBuilder( storeDirectory, fs, pageCache ) + .withConfig( config ).build(); LogTailScanner tailScanner = new LogTailScanner( logFiles, new VersionAwareLogEntryReader<>(), monitors ); // Add the kernel store migrator