diff --git a/community/command-line/src/main/java/org/neo4j/commandline/Util.java b/community/command-line/src/main/java/org/neo4j/commandline/Util.java index cbbaccfa0c2dc..d724e529b0ed5 100644 --- a/community/command-line/src/main/java/org/neo4j/commandline/Util.java +++ b/community/command-line/src/main/java/org/neo4j/commandline/Util.java @@ -63,6 +63,20 @@ public static Path canonicalPath( File file ) throws IllegalArgumentException } } + public static boolean isSameOrChildFile( File parent, File candidate ) + { + Path canonicalCandidate = canonicalPath( candidate ); + Path canonicalParentPath = canonicalPath( parent ); + return canonicalCandidate.startsWith( canonicalParentPath ); + } + + public static boolean isSameOrChildPath( Path parent, Path candidate ) + { + Path normalizedCandidate = candidate.normalize(); + Path normalizedParent = parent.normalize(); + return normalizedCandidate.startsWith( normalizedParent ); + } + public static void checkLock( Path databaseDirectory ) throws CommandFailed { try ( FileSystemAbstraction fileSystem = new DefaultFileSystemAbstraction(); diff --git a/community/consistency-check/src/main/java/org/neo4j/consistency/CheckConsistencyCommand.java b/community/consistency-check/src/main/java/org/neo4j/consistency/CheckConsistencyCommand.java index 5b63b6cf2d5c1..9255fd0d076df 100644 --- a/community/consistency-check/src/main/java/org/neo4j/consistency/CheckConsistencyCommand.java +++ b/community/consistency-check/src/main/java/org/neo4j/consistency/CheckConsistencyCommand.java @@ -36,7 +36,6 @@ import org.neo4j.commandline.arguments.common.OptionalCanonicalPath; import org.neo4j.consistency.checking.full.ConsistencyCheckIncompleteException; import org.neo4j.consistency.checking.full.ConsistencyFlags; -import org.neo4j.dbms.DatabaseManagementSystemSettings; import org.neo4j.graphdb.factory.GraphDatabaseSettings; import org.neo4j.helpers.Strings; import org.neo4j.helpers.collection.MapUtil; @@ -51,7 +50,7 @@ import org.neo4j.logging.FormattedLogProvider; import static java.lang.String.format; -import static org.neo4j.dbms.DatabaseManagementSystemSettings.database_path; +import static org.neo4j.graphdb.factory.GraphDatabaseSettings.database_path; public class CheckConsistencyCommand implements AdminCommand { @@ -234,7 +233,7 @@ private void checkDbState( File storeDir, Config additionalConfiguration ) throw .createPageCache( fileSystem, additionalConfiguration ) ) { RecoveryRequiredChecker requiredChecker = - new RecoveryRequiredChecker( fileSystem, pageCache, new Monitors() ); + new RecoveryRequiredChecker( fileSystem, pageCache, additionalConfiguration, new Monitors() ); if ( requiredChecker.isRecoveryRequiredAt( storeDir ) ) { throw new CommandFailed( @@ -253,7 +252,7 @@ private void checkDbState( File storeDir, Config additionalConfiguration ) throw private static Config loadNeo4jConfig( Path homeDir, Path configDir, String databaseName, Map additionalConfig ) { - additionalConfig.put( DatabaseManagementSystemSettings.active_database.name(), databaseName ); + additionalConfig.put( GraphDatabaseSettings.active_database.name(), databaseName ); return Config.fromFile( configDir.resolve( Config.DEFAULT_CONFIG_FILE_NAME ) ).withHome( homeDir ).withConnectorsDisabled() .withSettings( additionalConfig ).build(); diff --git a/community/consistency-check/src/main/java/org/neo4j/consistency/ConsistencyCheckTool.java b/community/consistency-check/src/main/java/org/neo4j/consistency/ConsistencyCheckTool.java index e910ed152b4c9..16e9b21afb262 100644 --- a/community/consistency-check/src/main/java/org/neo4j/consistency/ConsistencyCheckTool.java +++ b/community/consistency-check/src/main/java/org/neo4j/consistency/ConsistencyCheckTool.java @@ -26,8 +26,8 @@ import java.util.List; import java.util.Map; -import org.neo4j.consistency.checking.full.ConsistencyFlags; import org.neo4j.consistency.checking.full.ConsistencyCheckIncompleteException; +import org.neo4j.consistency.checking.full.ConsistencyFlags; import org.neo4j.graphdb.factory.GraphDatabaseSettings; import org.neo4j.helpers.Args; import org.neo4j.helpers.Strings; @@ -137,7 +137,8 @@ private void checkDbState( File storeDir, Config tuningConfiguration ) throws To { try ( PageCache pageCache = ConfigurableStandalonePageCacheFactory.createPageCache( fs, tuningConfiguration ) ) { - RecoveryRequiredChecker requiredChecker = new RecoveryRequiredChecker( fs, pageCache, new Monitors() ); + RecoveryRequiredChecker requiredChecker = new RecoveryRequiredChecker( fs, pageCache, + tuningConfiguration, new Monitors() ); if ( requiredChecker.isRecoveryRequiredAt( storeDir ) ) { throw new ToolFailureException( Strings.joinAsLines( diff --git a/community/consistency-check/src/test/java/org/neo4j/consistency/ConsistencyCheckToolTest.java b/community/consistency-check/src/test/java/org/neo4j/consistency/ConsistencyCheckToolTest.java index ef9a1c9fa23b9..d3b672cb06859 100644 --- a/community/consistency-check/src/test/java/org/neo4j/consistency/ConsistencyCheckToolTest.java +++ b/community/consistency-check/src/test/java/org/neo4j/consistency/ConsistencyCheckToolTest.java @@ -43,6 +43,7 @@ import org.neo4j.graphdb.GraphDatabaseService; import org.neo4j.graphdb.Transaction; import org.neo4j.graphdb.factory.GraphDatabaseSettings; +import org.neo4j.helpers.collection.MapUtil; import org.neo4j.helpers.progress.ProgressMonitorFactory; import org.neo4j.io.fs.DefaultFileSystemAbstraction; import org.neo4j.io.fs.FileSystemAbstraction; @@ -66,20 +67,21 @@ import static org.mockito.Mockito.verify; import static org.mockito.Mockito.verifyZeroInteractions; import static org.neo4j.graphdb.Label.label; +import static org.neo4j.graphdb.factory.GraphDatabaseSettings.logical_logs_location; public class ConsistencyCheckToolTest { - private final TestDirectory storeDirectory = TestDirectory.testDirectory(); + private final TestDirectory testDirectory = TestDirectory.testDirectory(); private final EphemeralFileSystemRule fs = new EphemeralFileSystemRule(); @Rule - public RuleChain ruleChain = RuleChain.outerRule( storeDirectory ).around( fs ); + public RuleChain ruleChain = RuleChain.outerRule( testDirectory ).around( fs ); @Test public void runsConsistencyCheck() throws Exception { // given - File storeDir = storeDirectory.directory(); + File storeDir = testDirectory.directory(); String[] args = {storeDir.getPath()}; ConsistencyCheckService service = mock( ConsistencyCheckService.class ); @@ -108,8 +110,8 @@ public void consistencyCheckerLogUseSystemTimezoneIfConfigurable() throws Except provider.getLog( "test" ).info( "testMessage" ); return ConsistencyCheckService.Result.success( new File( StringUtils.EMPTY ) ); } ); - File storeDir = storeDirectory.directory(); - File configFile = storeDirectory.file( Config.DEFAULT_CONFIG_FILE_NAME ); + File storeDir = testDirectory.directory(); + File configFile = testDirectory.file( Config.DEFAULT_CONFIG_FILE_NAME ); Properties properties = new Properties(); properties.setProperty( GraphDatabaseSettings.log_timezone.name(), LogTimeZone.SYSTEM.name() ); properties.store( new FileWriter( configFile ), null ); @@ -128,7 +130,7 @@ public void consistencyCheckerLogUseSystemTimezoneIfConfigurable() throws Except public void appliesDefaultTuningConfigurationForConsistencyChecker() throws Exception { // given - File storeDir = storeDirectory.directory(); + File storeDir = testDirectory.directory(); String[] args = {storeDir.getPath()}; ConsistencyCheckService service = mock( ConsistencyCheckService.class ); @@ -147,8 +149,8 @@ public void appliesDefaultTuningConfigurationForConsistencyChecker() throws Exce public void passesOnConfigurationIfProvided() throws Exception { // given - File storeDir = storeDirectory.directory(); - File configFile = storeDirectory.file( Config.DEFAULT_CONFIG_FILE_NAME ); + File storeDir = testDirectory.directory(); + File configFile = testDirectory.file( Config.DEFAULT_CONFIG_FILE_NAME ); Properties properties = new Properties(); properties.setProperty( ConsistencyCheckSettings.consistency_check_property_owners.name(), "true" ); properties.store( new FileWriter( configFile ), null ); @@ -191,8 +193,8 @@ public void exitWithFailureIndicatingCorrectUsageIfNoArgumentsSupplied() throws public void exitWithFailureIfConfigSpecifiedButConfigFileDoesNotExist() throws Exception { // given - File configFile = storeDirectory.file( "nonexistent_file" ); - String[] args = {storeDirectory.directory().getPath(), "-config", configFile.getPath()}; + File configFile = testDirectory.file( "nonexistent_file" ); + String[] args = {testDirectory.directory().getPath(), "-config", configFile.getPath()}; ConsistencyCheckService service = mock( ConsistencyCheckService.class ); try @@ -214,9 +216,34 @@ public void exitWithFailureIfConfigSpecifiedButConfigFileDoesNotExist() throws E @Test( expected = ToolFailureException.class ) public void failWhenStoreWasNonCleanlyShutdown() throws Exception { - createGraphDbAndKillIt(); + createGraphDbAndKillIt( Config.defaults() ); - runConsistencyCheckToolWith( fs.get(), storeDirectory.graphDbDir().getAbsolutePath() ); + runConsistencyCheckToolWith( fs.get(), testDirectory.graphDbDir().getAbsolutePath() ); + } + + @Test( expected = ToolFailureException.class ) + public void failOnNotCleanlyShutdownStoreWithLogsInCustomRelativeLocation() throws Exception + { + File customConfigFile = testDirectory.file( "customConfig" ); + Config customConfig = Config.defaults( logical_logs_location, "otherLocation" ); + createGraphDbAndKillIt( customConfig ); + MapUtil.store( customConfig.getRaw(), customConfigFile ); + String[] args = {testDirectory.graphDbDir().getPath(), "-config", customConfigFile.getPath()}; + + runConsistencyCheckToolWith( fs.get(), args ); + } + + @Test( expected = ToolFailureException.class ) + public void failOnNotCleanlyShutdownStoreWithLogsInCustomAbsoluteLocation() throws Exception + { + File customConfigFile = testDirectory.file( "customConfig" ); + File otherLocation = testDirectory.directory( "otherLocation" ); + Config customConfig = Config.defaults( logical_logs_location, otherLocation.getAbsolutePath() ); + createGraphDbAndKillIt( customConfig ); + MapUtil.store( customConfig.getRaw(), customConfigFile ); + String[] args = {testDirectory.graphDbDir().getPath(), "-config", customConfigFile.getPath()}; + + runConsistencyCheckToolWith( fs.get(), args ); } private void checkLogRecordTimeZone( ConsistencyCheckService service, String[] args, int hoursShift, @@ -237,11 +264,12 @@ private String readLogLine( ByteArrayOutputStream outputStream ) throws IOExcept return bufferedReader.readLine(); } - private void createGraphDbAndKillIt() throws Exception + private void createGraphDbAndKillIt( Config config ) throws Exception { final GraphDatabaseService db = new TestGraphDatabaseFactory() .setFileSystem( fs.get() ) - .newImpermanentDatabaseBuilder( storeDirectory.graphDbDir() ) + .newImpermanentDatabaseBuilder( testDirectory.graphDbDir() ) + .setConfig( config.getRaw() ) .newGraphDatabase(); try ( Transaction tx = db.beginTx() ) diff --git a/community/dbms/src/main/java/org/neo4j/commandline/dbms/CsvImporter.java b/community/dbms/src/main/java/org/neo4j/commandline/dbms/CsvImporter.java index d30861ffae8f7..75974c7d1d9c7 100644 --- a/community/dbms/src/main/java/org/neo4j/commandline/dbms/CsvImporter.java +++ b/community/dbms/src/main/java/org/neo4j/commandline/dbms/CsvImporter.java @@ -30,7 +30,6 @@ import org.neo4j.commandline.admin.OutsideWorld; import org.neo4j.commandline.dbms.config.WrappedBatchImporterConfigurationForNeo4jAdmin; import org.neo4j.commandline.dbms.config.WrappedCsvInputConfigurationForNeo4jAdmin; -import org.neo4j.dbms.DatabaseManagementSystemSettings; import org.neo4j.graphdb.factory.GraphDatabaseSettings; import org.neo4j.helpers.Args; import org.neo4j.io.fs.FileSystemAbstraction; @@ -43,7 +42,6 @@ import org.neo4j.unsafe.impl.batchimport.input.csv.IdType; import static java.nio.charset.Charset.defaultCharset; - import static org.neo4j.kernel.impl.util.Converters.withDefault; import static org.neo4j.tooling.ImportTool.csvConfiguration; import static org.neo4j.tooling.ImportTool.extractInputFiles; @@ -100,7 +98,7 @@ class CsvImporter implements Importer public void doImport() throws IOException { FileSystemAbstraction fs = outsideWorld.fileSystem(); - File storeDir = databaseConfig.get( DatabaseManagementSystemSettings.database_path ); + File storeDir = databaseConfig.get( GraphDatabaseSettings.database_path ); File logsDir = databaseConfig.get( GraphDatabaseSettings.logs_directory ); File reportFile = new File( reportFileName ); diff --git a/community/dbms/src/main/java/org/neo4j/commandline/dbms/DatabaseImporter.java b/community/dbms/src/main/java/org/neo4j/commandline/dbms/DatabaseImporter.java index a539954720b2a..d126a3197df05 100644 --- a/community/dbms/src/main/java/org/neo4j/commandline/dbms/DatabaseImporter.java +++ b/community/dbms/src/main/java/org/neo4j/commandline/dbms/DatabaseImporter.java @@ -30,7 +30,7 @@ import org.neo4j.kernel.impl.util.Converters; import org.neo4j.kernel.impl.util.Validators; -import static org.neo4j.dbms.DatabaseManagementSystemSettings.database_path; +import static org.neo4j.graphdb.factory.GraphDatabaseSettings.database_path; class DatabaseImporter implements Importer { diff --git a/community/dbms/src/main/java/org/neo4j/commandline/dbms/DumpCommand.java b/community/dbms/src/main/java/org/neo4j/commandline/dbms/DumpCommand.java index a7d3e775e92b3..4549164726262 100644 --- a/community/dbms/src/main/java/org/neo4j/commandline/dbms/DumpCommand.java +++ b/community/dbms/src/main/java/org/neo4j/commandline/dbms/DumpCommand.java @@ -32,8 +32,8 @@ import org.neo4j.commandline.admin.CommandFailed; import org.neo4j.commandline.admin.IncorrectUsage; import org.neo4j.commandline.arguments.Arguments; -import org.neo4j.dbms.DatabaseManagementSystemSettings; import org.neo4j.dbms.archive.Dumper; +import org.neo4j.graphdb.factory.GraphDatabaseSettings; import org.neo4j.kernel.StoreLockException; import org.neo4j.kernel.configuration.Config; import org.neo4j.kernel.impl.util.Validators; @@ -41,7 +41,8 @@ import static java.lang.String.format; import static org.neo4j.commandline.Util.canonicalPath; -import static org.neo4j.dbms.DatabaseManagementSystemSettings.database_path; +import static org.neo4j.graphdb.factory.GraphDatabaseSettings.database_path; +import static org.neo4j.graphdb.factory.GraphDatabaseSettings.logical_logs_location; public class DumpCommand implements AdminCommand { @@ -66,7 +67,10 @@ public void execute( String[] args ) throws IncorrectUsage, CommandFailed { String database = arguments.parse( args ).get( "database" ); Path archive = calculateArchive( database, arguments.getMandatoryPath( "to" ) ); - Path databaseDirectory = canonicalPath( toDatabaseDirectory( database ) ); + + Config config = buildConfig( database ); + Path databaseDirectory = canonicalPath( getDatabaseDirectory( config ) ); + Path transactionLogsDirectory = canonicalPath( getTransactionalLogsDirectory( config ) ); try { @@ -79,7 +83,7 @@ public void execute( String[] args ) throws IncorrectUsage, CommandFailed try ( Closeable ignored = StoreLockChecker.check( databaseDirectory ) ) { - dump( database, databaseDirectory, archive ); + dump( database, databaseDirectory, transactionLogsDirectory, archive ); } catch ( StoreLockException e ) { @@ -97,13 +101,23 @@ public void execute( String[] args ) throws IncorrectUsage, CommandFailed } } - private Path toDatabaseDirectory( String databaseName ) + private Path getDatabaseDirectory( Config config ) + { + return config.get( database_path ).toPath(); + } + + private Path getTransactionalLogsDirectory( Config config ) + { + return config.get( logical_logs_location ).toPath(); + } + + private Config buildConfig( String databaseName ) { return Config.fromFile( configDir.resolve( Config.DEFAULT_CONFIG_FILE_NAME ) ) .withHome( homeDir ) .withConnectorsDisabled() - .withSetting( DatabaseManagementSystemSettings.active_database, databaseName ) - .build().get( database_path ).toPath(); + .withSetting( GraphDatabaseSettings.active_database, databaseName ) + .build(); } private Path calculateArchive( String database, Path to ) @@ -111,11 +125,12 @@ private Path calculateArchive( String database, Path to ) return Files.isDirectory( to ) ? to.resolve( database + ".dump" ) : to; } - private void dump( String database, Path databaseDirectory, Path archive ) throws CommandFailed + private void dump( String database, Path databaseDirectory, Path transactionalLogsDirectory, Path archive ) + throws CommandFailed { try { - dumper.dump( databaseDirectory, archive, this::isStoreLock ); + dumper.dump( databaseDirectory, transactionalLogsDirectory, archive, this::isStoreLock ); } catch ( FileAlreadyExistsException e ) { diff --git a/community/dbms/src/main/java/org/neo4j/commandline/dbms/ImportCommand.java b/community/dbms/src/main/java/org/neo4j/commandline/dbms/ImportCommand.java index 08a2d53181589..2eb17447511a2 100644 --- a/community/dbms/src/main/java/org/neo4j/commandline/dbms/ImportCommand.java +++ b/community/dbms/src/main/java/org/neo4j/commandline/dbms/ImportCommand.java @@ -34,7 +34,7 @@ import org.neo4j.commandline.arguments.OptionalBooleanArg; import org.neo4j.commandline.arguments.OptionalNamedArg; import org.neo4j.commandline.arguments.OptionalNamedArgWithMetadata; -import org.neo4j.dbms.DatabaseManagementSystemSettings; +import org.neo4j.graphdb.factory.GraphDatabaseSettings; import org.neo4j.helpers.Args; import org.neo4j.helpers.collection.MapUtil; import org.neo4j.kernel.configuration.Config; @@ -250,7 +250,7 @@ public void execute( String[] args ) throws IncorrectUsage, CommandFailed Config config = loadNeo4jConfig( homeDir, configDir, database, loadAdditionalConfig( additionalConfigFile ) ); Validators.CONTAINS_NO_EXISTING_DATABASE - .validate( config.get( DatabaseManagementSystemSettings.database_path ) ); + .validate( config.get( GraphDatabaseSettings.database_path ) ); Importer importer = importerFactory.getImporterForMode( mode, Args.parse( args ), config, outsideWorld ); importer.doImport(); @@ -288,7 +288,7 @@ private static Config loadNeo4jConfig( Path homeDir, Path configDir, String data { return Config.fromFile( configDir.resolve( Config.DEFAULT_CONFIG_FILE_NAME ) ) .withHome( homeDir ) - .withSetting( DatabaseManagementSystemSettings.active_database, databaseName ) + .withSetting( GraphDatabaseSettings.active_database, databaseName ) .withSettings( additionalConfig ) .withConnectorsDisabled().build(); } diff --git a/community/dbms/src/main/java/org/neo4j/commandline/dbms/LoadCommand.java b/community/dbms/src/main/java/org/neo4j/commandline/dbms/LoadCommand.java index 8538c6fdc36a7..914e0aee7f6f4 100644 --- a/community/dbms/src/main/java/org/neo4j/commandline/dbms/LoadCommand.java +++ b/community/dbms/src/main/java/org/neo4j/commandline/dbms/LoadCommand.java @@ -32,17 +32,19 @@ import org.neo4j.commandline.arguments.Arguments; import org.neo4j.commandline.arguments.OptionalBooleanArg; import org.neo4j.commandline.arguments.common.MandatoryCanonicalPath; -import org.neo4j.dbms.DatabaseManagementSystemSettings; import org.neo4j.dbms.archive.IncorrectFormat; import org.neo4j.dbms.archive.Loader; +import org.neo4j.graphdb.factory.GraphDatabaseSettings; import org.neo4j.io.fs.FileUtils; import org.neo4j.kernel.configuration.Config; import static java.util.Objects.requireNonNull; import static org.neo4j.commandline.Util.canonicalPath; import static org.neo4j.commandline.Util.checkLock; +import static org.neo4j.commandline.Util.isSameOrChildPath; import static org.neo4j.commandline.Util.wrapIOException; -import static org.neo4j.dbms.DatabaseManagementSystemSettings.database_path; +import static org.neo4j.graphdb.factory.GraphDatabaseSettings.database_path; +import static org.neo4j.graphdb.factory.GraphDatabaseSettings.logical_logs_location; public class LoadCommand implements AdminCommand { @@ -74,22 +76,35 @@ public void execute( String[] args ) throws IncorrectUsage, CommandFailed String database = arguments.get( "database" ); boolean force = arguments.getBoolean( "force" ); - Path databaseDirectory = canonicalPath( toDatabaseDirectory( database ) ); + Config config = buildConfig( database ); - deleteIfNecessary( databaseDirectory, force ); - load( archive, database, databaseDirectory ); + Path databaseDirectory = canonicalPath( getDatabaseDirectory( config ) ); + Path transactionLogsDirectory = canonicalPath( getTransactionalLogsDirectory( config ) ); + + deleteIfNecessary( databaseDirectory, transactionLogsDirectory, force ); + load( archive, database, databaseDirectory, transactionLogsDirectory ); + } + + private Path getDatabaseDirectory( Config config ) + { + return config.get( database_path ).toPath(); + } + + private Path getTransactionalLogsDirectory( Config config ) + { + return config.get( logical_logs_location ).toPath(); } - private Path toDatabaseDirectory( String databaseName ) + private Config buildConfig( String databaseName ) { return Config.fromFile( configDir.resolve( Config.DEFAULT_CONFIG_FILE_NAME ) ) .withHome( homeDir ) .withConnectorsDisabled() - .withSetting( DatabaseManagementSystemSettings.active_database, databaseName ) - .build().get( database_path ).toPath(); + .withSetting( GraphDatabaseSettings.active_database, databaseName ) + .build(); } - private void deleteIfNecessary( Path databaseDirectory, boolean force ) throws CommandFailed + private void deleteIfNecessary( Path databaseDirectory, Path transactionLogsDirectory, boolean force ) throws CommandFailed { try { @@ -97,6 +112,10 @@ private void deleteIfNecessary( Path databaseDirectory, boolean force ) throws C { checkLock( databaseDirectory ); FileUtils.deletePathRecursively( databaseDirectory ); + if ( !isSameOrChildPath( databaseDirectory, transactionLogsDirectory ) ) + { + FileUtils.deletePathRecursively( transactionLogsDirectory ); + } } } catch ( IOException e ) @@ -105,11 +124,11 @@ private void deleteIfNecessary( Path databaseDirectory, boolean force ) throws C } } - private void load( Path archive, String database, Path databaseDirectory ) throws CommandFailed + private void load( Path archive, String database, Path databaseDirectory, Path transactionLogsDirectory ) throws CommandFailed { try { - loader.load( archive, databaseDirectory ); + loader.load( archive, databaseDirectory, transactionLogsDirectory ); } catch ( NoSuchFileException e ) { diff --git a/community/dbms/src/main/java/org/neo4j/dbms/DatabaseManagementSystemSettings.java b/community/dbms/src/main/java/org/neo4j/dbms/DatabaseManagementSystemSettings.java index 74c478c9e951c..37ceb2bdbde1f 100644 --- a/community/dbms/src/main/java/org/neo4j/dbms/DatabaseManagementSystemSettings.java +++ b/community/dbms/src/main/java/org/neo4j/dbms/DatabaseManagementSystemSettings.java @@ -21,35 +21,19 @@ import java.io.File; -import org.neo4j.configuration.Description; import org.neo4j.configuration.Internal; import org.neo4j.configuration.LoadableConfig; import org.neo4j.graphdb.config.Setting; +import org.neo4j.graphdb.factory.GraphDatabaseSettings; import static org.neo4j.kernel.configuration.Settings.PATH; -import static org.neo4j.kernel.configuration.Settings.STRING; import static org.neo4j.kernel.configuration.Settings.derivedSetting; -import static org.neo4j.kernel.configuration.Settings.pathSetting; -import static org.neo4j.kernel.configuration.Settings.setting; public class DatabaseManagementSystemSettings implements LoadableConfig { - @Description( "Name of the database to load" ) - public static final Setting active_database = setting( "dbms.active_database", STRING, "graph.db" ); - - @Description( "Path of the data directory. You must not configure more than one Neo4j installation to use the " + - "same data directory." ) - public static final Setting data_directory = pathSetting( "dbms.directories.data", "data" ); - - @Internal - public static final Setting database_path = derivedSetting( "unsupported.dbms.directories.database", - data_directory, active_database, - ( data, current ) -> new File( new File( data, "databases" ), current ), - PATH ); - @Internal public static final Setting auth_store_directory = derivedSetting( "unsupported.dbms.directories.auth", - data_directory, + GraphDatabaseSettings.data_directory, data -> new File( data, "dbms" ), PATH ); } diff --git a/community/dbms/src/main/java/org/neo4j/dbms/archive/Dumper.java b/community/dbms/src/main/java/org/neo4j/dbms/archive/Dumper.java index 69e471f3eb142..e7152d3b20adc 100644 --- a/community/dbms/src/main/java/org/neo4j/dbms/archive/Dumper.java +++ b/community/dbms/src/main/java/org/neo4j/dbms/archive/Dumper.java @@ -19,6 +19,11 @@ */ package org.neo4j.dbms.archive; +import org.apache.commons.compress.archivers.ArchiveEntry; +import org.apache.commons.compress.archivers.ArchiveOutputStream; +import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream; +import org.apache.commons.compress.compressors.gzip.GzipCompressorOutputStream; + import java.io.IOException; import java.io.InputStream; import java.nio.file.Files; @@ -26,11 +31,7 @@ import java.nio.file.StandardOpenOption; import java.util.function.Predicate; -import org.apache.commons.compress.archivers.ArchiveEntry; -import org.apache.commons.compress.archivers.ArchiveOutputStream; -import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream; -import org.apache.commons.compress.compressors.gzip.GzipCompressorOutputStream; - +import org.neo4j.commandline.Util; import org.neo4j.function.ThrowingAction; import static org.neo4j.dbms.archive.Utils.checkWritableDirectory; @@ -45,20 +46,31 @@ public class Dumper { - public void dump( Path root, Path archive, Predicate exclude ) throws IOException + public void dump( Path dbPath, Path transactionalLogsPath, Path archive, Predicate exclude ) + throws IOException { checkWritableDirectory( archive.getParent() ); try ( ArchiveOutputStream stream = openArchiveOut( archive ) ) { - Files.walkFileTree( root, - onlyMatching( not( exclude ), - throwExceptions( - onDirectory( dir -> dumpDirectory( root, stream, dir ), - onFile( file -> dumpFile( root, stream, file ), - justContinue() ) ) ) ) ); + visitPath( dbPath, exclude, stream ); + if ( !Util.isSameOrChildPath( dbPath, transactionalLogsPath ) ) + { + visitPath( transactionalLogsPath, exclude, stream ); + } } } + private void visitPath( Path transactionalLogsPath, Predicate exclude, ArchiveOutputStream stream ) + throws IOException + { + Files.walkFileTree( transactionalLogsPath, + onlyMatching( not( exclude ), + throwExceptions( + onDirectory( dir -> dumpDirectory( transactionalLogsPath, stream, dir ), + onFile( file -> dumpFile( transactionalLogsPath, stream, file ), + justContinue() ) ) ) ) ); + } + private static ArchiveOutputStream openArchiveOut( Path archive ) throws IOException { // StandardOpenOption.CREATE_NEW is important here because it atomically asserts that the file doesn't diff --git a/community/dbms/src/main/java/org/neo4j/dbms/archive/Loader.java b/community/dbms/src/main/java/org/neo4j/dbms/archive/Loader.java index a9113eb09dc7b..e6fa932c1af8c 100644 --- a/community/dbms/src/main/java/org/neo4j/dbms/archive/Loader.java +++ b/community/dbms/src/main/java/org/neo4j/dbms/archive/Loader.java @@ -19,41 +19,71 @@ */ package org.neo4j.dbms.archive; +import org.apache.commons.compress.archivers.ArchiveEntry; +import org.apache.commons.compress.archivers.ArchiveInputStream; +import org.apache.commons.compress.archivers.tar.TarArchiveInputStream; +import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream; + import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; import java.nio.file.FileAlreadyExistsException; +import java.nio.file.FileSystemException; import java.nio.file.Files; import java.nio.file.Path; +import java.nio.file.Paths; -import org.apache.commons.compress.archivers.ArchiveEntry; -import org.apache.commons.compress.archivers.ArchiveInputStream; -import org.apache.commons.compress.archivers.tar.TarArchiveInputStream; -import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream; +import org.neo4j.kernel.impl.transaction.log.files.TransactionLogFiles; import static java.nio.file.Files.exists; - import static org.neo4j.dbms.archive.Utils.checkWritableDirectory; public class Loader { - public void load( Path archive, Path destination ) throws IOException, IncorrectFormat + public void load( Path archive, Path databaseDestination, Path transactionLogsDirectory ) throws IOException, IncorrectFormat { - if ( exists( destination ) ) - { - throw new FileAlreadyExistsException( destination.toString() ); - } - checkWritableDirectory( destination.getParent() ); + validatePath( databaseDestination ); + validatePath( transactionLogsDirectory ); + + createDestination( databaseDestination ); + createDestination( transactionLogsDirectory ); + try ( ArchiveInputStream stream = openArchiveIn( archive ) ) { ArchiveEntry entry; while ( (entry = nextEntry( stream, archive )) != null ) { + Path destination = determineEntryDestination( entry, databaseDestination, transactionLogsDirectory ); loadEntry( destination, stream, entry ); } } } + private void createDestination( Path destination ) throws IOException + { + if ( !destination.toFile().exists() ) + { + Files.createDirectories( destination ); + } + } + + private void validatePath( Path path ) throws FileSystemException + { + if ( exists( path ) ) + { + throw new FileAlreadyExistsException( path.toString() ); + } + checkWritableDirectory( path.getParent() ); + } + + private static Path determineEntryDestination( ArchiveEntry entry, Path databaseDestination, + Path transactionLogsDirectory ) + { + String entryName = Paths.get( entry.getName() ).getFileName().toString(); + return TransactionLogFiles.DEFAULT_FILENAME_FILTER.accept( null, entryName ) ? transactionLogsDirectory + : databaseDestination; + } + private ArchiveEntry nextEntry( ArchiveInputStream stream, Path archive ) throws IncorrectFormat { try diff --git a/community/dbms/src/test/java/org/neo4j/commandline/dbms/CsvImporterTest.java b/community/dbms/src/test/java/org/neo4j/commandline/dbms/CsvImporterTest.java index e288fefe78194..d5a28df9cace2 100644 --- a/community/dbms/src/test/java/org/neo4j/commandline/dbms/CsvImporterTest.java +++ b/community/dbms/src/test/java/org/neo4j/commandline/dbms/CsvImporterTest.java @@ -30,7 +30,6 @@ import java.util.Map; import org.neo4j.commandline.admin.RealOutsideWorld; -import org.neo4j.dbms.DatabaseManagementSystemSettings; import org.neo4j.graphdb.factory.GraphDatabaseSettings; import org.neo4j.helpers.Args; import org.neo4j.kernel.configuration.Config; @@ -62,7 +61,7 @@ public void writesReportToSpecifiedReportFile() throws Exception { Config config = Config.builder() .withSettings( additionalConfig() ) - .withSetting( DatabaseManagementSystemSettings.database_path, dbDir.getAbsolutePath() ) + .withSetting( GraphDatabaseSettings.database_path, dbDir.getAbsolutePath() ) .withSetting( GraphDatabaseSettings.logs_directory, logDir.getAbsolutePath() ).build(); CsvImporter csvImporter = new CsvImporter( @@ -80,7 +79,7 @@ public void writesReportToSpecifiedReportFile() throws Exception private Map additionalConfig() { - return stringMap( DatabaseManagementSystemSettings.database_path.name(), getDatabasePath(), + return stringMap( GraphDatabaseSettings.database_path.name(), getDatabasePath(), GraphDatabaseSettings.logs_directory.name(), getLogsDirectory() ); } diff --git a/community/dbms/src/test/java/org/neo4j/commandline/dbms/DatabaseImporterTest.java b/community/dbms/src/test/java/org/neo4j/commandline/dbms/DatabaseImporterTest.java index 4eec3d376b8f0..432e8d763b1cc 100644 --- a/community/dbms/src/test/java/org/neo4j/commandline/dbms/DatabaseImporterTest.java +++ b/community/dbms/src/test/java/org/neo4j/commandline/dbms/DatabaseImporterTest.java @@ -30,7 +30,6 @@ import org.neo4j.commandline.admin.IncorrectUsage; import org.neo4j.commandline.admin.NullOutsideWorld; -import org.neo4j.dbms.DatabaseManagementSystemSettings; import org.neo4j.graphdb.GraphDatabaseService; import org.neo4j.graphdb.Transaction; import org.neo4j.graphdb.factory.GraphDatabaseSettings; @@ -128,7 +127,7 @@ private Config getConfigWith( File homeDir, String databaseName ) { HashMap additionalConfig = new HashMap<>(); additionalConfig.put( GraphDatabaseSettings.neo4j_home.name(), homeDir.toString() ); - additionalConfig.put( DatabaseManagementSystemSettings.active_database.name(), databaseName ); + additionalConfig.put( GraphDatabaseSettings.active_database.name(), databaseName ); return Config.defaults( additionalConfig ); } diff --git a/community/dbms/src/test/java/org/neo4j/commandline/dbms/DumpCommandTest.java b/community/dbms/src/test/java/org/neo4j/commandline/dbms/DumpCommandTest.java index d9c89cf8a3464..c58b7418025cb 100644 --- a/community/dbms/src/test/java/org/neo4j/commandline/dbms/DumpCommandTest.java +++ b/community/dbms/src/test/java/org/neo4j/commandline/dbms/DumpCommandTest.java @@ -43,6 +43,7 @@ import org.neo4j.commandline.admin.IncorrectUsage; import org.neo4j.commandline.admin.Usage; import org.neo4j.dbms.archive.Dumper; +import org.neo4j.graphdb.config.Setting; import org.neo4j.io.fs.DefaultFileSystemAbstraction; import org.neo4j.io.fs.FileSystemAbstraction; import org.neo4j.kernel.configuration.Config; @@ -66,8 +67,9 @@ import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; -import static org.neo4j.dbms.DatabaseManagementSystemSettings.data_directory; import static org.neo4j.dbms.archive.TestUtils.withPermissions; +import static org.neo4j.graphdb.factory.GraphDatabaseSettings.data_directory; +import static org.neo4j.graphdb.factory.GraphDatabaseSettings.logical_logs_location; public class DumpCommandTest { @@ -97,7 +99,8 @@ public void setUp() throws Exception public void shouldDumpTheDatabaseToTheArchive() throws Exception { execute( "foo.db" ); - verify( dumper ).dump( eq( homeDir.resolve( "data/databases/foo.db" ) ), eq( archive ), any() ); + verify( dumper ).dump( eq( homeDir.resolve( "data/databases/foo.db" ) ), + eq( homeDir.resolve( "data/databases/foo.db" ) ), eq( archive ), any() ); } @Test @@ -107,10 +110,25 @@ public void shouldCalculateTheDatabaseDirectoryFromConfig() throws Exception Path databaseDir = dataDir.resolve( "databases/foo.db" ); putStoreInDirectory( databaseDir ); Files.write( configDir.resolve( Config.DEFAULT_CONFIG_FILE_NAME ), - asList( format( "%s=%s", data_directory.name(), dataDir.toString().replace( '\\', '/' ) ) ) ); + asList( formatProperty( data_directory, dataDir ) ) ); execute( "foo.db" ); - verify( dumper ).dump( eq( databaseDir ), any(), any() ); + verify( dumper ).dump( eq( databaseDir ), eq( databaseDir ), any(), any() ); + } + + @Test + public void shouldCalculateTheTxLogDirectoryFromConfig() throws Exception + { + Path dataDir = testDirectory.directory( "some-other-path" ).toPath(); + Path txLogsDir = testDirectory.directory( "txLogsPath" ).toPath(); + Path databaseDir = dataDir.resolve( "databases/foo.db" ); + putStoreInDirectory( databaseDir ); + Files.write( configDir.resolve( Config.DEFAULT_CONFIG_FILE_NAME ), + asList( formatProperty( data_directory, dataDir ), + formatProperty( logical_logs_location, txLogsDir ) ) ); + + execute( "foo.db" ); + verify( dumper ).dump( eq( databaseDir ), eq( txLogsDir ), any(), any() ); } @Test @@ -132,7 +150,7 @@ public void shouldHandleDatabaseSymlink() throws Exception asList( format( "%s=%s", data_directory.name(), dataDir.toString().replace( '\\', '/' ) ) ) ); execute( "foo.db" ); - verify( dumper ).dump( eq( realDatabaseDir ), any(), any() ); + verify( dumper ).dump( eq( realDatabaseDir ), eq( realDatabaseDir ), any(), any() ); } @Test @@ -141,7 +159,7 @@ public void shouldCalculateTheArchiveNameIfPassedAnExistingDirectory() { File to = testDirectory.directory( "some-dir" ); new DumpCommand( homeDir, configDir, dumper ).execute( new String[]{"--database=" + "foo.db", "--to=" + to} ); - verify( dumper ).dump( any( Path.class ), eq( to.toPath().resolve( "foo.db.dump" ) ), any() ); + verify( dumper ).dump( any( Path.class ), any( Path.class ), eq( to.toPath().resolve( "foo.db.dump" ) ), any() ); } @Test @@ -149,7 +167,8 @@ public void shouldConvertToCanonicalPath() throws Exception { new DumpCommand( homeDir, configDir, dumper ) .execute( new String[]{"--database=" + "foo.db", "--to=foo.dump"} ); - verify( dumper ).dump( any( Path.class ), eq( Paths.get( new File( "foo.dump" ).getCanonicalPath() ) ), any() ); + verify( dumper ).dump( any( Path.class ), any( Path.class ), + eq( Paths.get( new File( "foo.dump" ).getCanonicalPath() ) ), any() ); } @Test @@ -158,7 +177,7 @@ public void shouldNotCalculateTheArchiveNameIfPassedAnExistingFile() { Files.createFile( archive ); execute( "foo.db" ); - verify( dumper ).dump( any(), eq( archive ), any() ); + verify( dumper ).dump( any(), any(), eq( archive ), any() ); } @Test @@ -190,7 +209,7 @@ public void shouldReleaseTheStoreLockAfterDumping() throws Exception @Test public void shouldReleaseTheStoreLockEvenIfThereIsAnError() throws Exception { - doThrow( IOException.class ).when( dumper ).dump( any(), any(), any() ); + doThrow( IOException.class ).when( dumper ).dump( any(), any(), any(), any() ); try { @@ -213,7 +232,7 @@ public void shouldNotAccidentallyCreateTheDatabaseDirectoryAsASideEffectOfStoreL { assertThat( Files.exists( databaseDirectory ), equalTo( false ) ); return null; - } ).when( dumper ).dump( any(), any(), any() ); + } ).when( dumper ).dump( any(), any(), any(), any() ); execute( "foo.db" ); } @@ -249,11 +268,11 @@ public void shouldExcludeTheStoreLockFromTheArchiveToAvoidProblemsWithReadingLoc doAnswer( invocation -> { //noinspection unchecked - Predicate exclude = invocation.getArgument( 2 ); + Predicate exclude = invocation.getArgument( 3 ); assertThat( exclude.test( Paths.get( StoreLocker.STORE_LOCK_FILENAME ) ), is( true ) ); assertThat( exclude.test( Paths.get( "some-other-file" ) ), is( false ) ); return null; - } ).when( dumper ).dump( any(), any(), any() ); + } ).when( dumper ).dump(any(), any(), any(), any() ); execute( "foo.db" ); } @@ -265,10 +284,10 @@ public void shouldDefaultToGraphDB() throws Exception Path databaseDir = dataDir.resolve( "databases/graph.db" ); putStoreInDirectory( databaseDir ); Files.write( configDir.resolve( Config.DEFAULT_CONFIG_FILE_NAME ), - asList( format( "%s=%s", data_directory.name(), dataDir.toString().replace( '\\', '/' ) ) ) ); + asList( formatProperty( data_directory, dataDir ) ) ); new DumpCommand( homeDir, configDir, dumper ).execute( new String[]{"--to=" + archive} ); - verify( dumper ).dump( eq( databaseDir ), any(), any() ); + verify( dumper ).dump( eq( databaseDir ), eq( databaseDir ), any(), any() ); } @Test @@ -288,7 +307,7 @@ public void shouldObjectIfTheArchiveArgumentIsMissing() throws Exception @Test public void shouldGiveAClearErrorIfTheArchiveAlreadyExists() throws Exception { - doThrow( new FileAlreadyExistsException( "the-archive-path" ) ).when( dumper ).dump( any(), any(), any() ); + doThrow( new FileAlreadyExistsException( "the-archive-path" ) ).when( dumper ).dump( any(), any(), any(), any() ); try { execute( "foo.db" ); @@ -317,7 +336,7 @@ public void shouldGiveAClearMessageIfTheDatabaseDoesntExist() throws Exception @Test public void shouldGiveAClearMessageIfTheArchivesParentDoesntExist() throws Exception { - doThrow( new NoSuchFileException( archive.getParent().toString() ) ).when( dumper ).dump( any(), any(), any() ); + doThrow( new NoSuchFileException( archive.getParent().toString() ) ).when( dumper ).dump(any(), any(), any(), any() ); try { execute( "foo.db" ); @@ -331,10 +350,10 @@ public void shouldGiveAClearMessageIfTheArchivesParentDoesntExist() throws Excep } @Test - public void shouldWrapIOExceptionsCarefulllyBecauseCriticalInformationIsOftenEncodedInTheirNameButMissingFromTheirMessage() + public void shouldWrapIOExceptionsCarefullyBecauseCriticalInformationIsOftenEncodedInTheirNameButMissingFromTheirMessage() throws Exception { - doThrow( new IOException( "the-message" ) ).when( dumper ).dump( any(), any(), any() ); + doThrow( new IOException( "the-message" ) ).when( dumper ).dump(any(), any(), any(), any() ); try { execute( "foo.db" ); @@ -383,7 +402,7 @@ private void execute( final String database ) throws IncorrectUsage, CommandFail .execute( new String[]{"--database=" + database, "--to=" + archive} ); } - private void assertCanLockStore( Path databaseDirectory ) throws IOException + private static void assertCanLockStore( Path databaseDirectory ) throws IOException { try ( FileSystemAbstraction fileSystem = new DefaultFileSystemAbstraction(); StoreLocker storeLocker = new StoreLocker( fileSystem, databaseDirectory.toFile() ) ) @@ -392,10 +411,15 @@ private void assertCanLockStore( Path databaseDirectory ) throws IOException } } - private void putStoreInDirectory( Path storeDir ) throws IOException + private static void putStoreInDirectory( Path storeDir ) throws IOException { Files.createDirectories( storeDir ); Path storeFile = storeDir.resolve( StoreFileType.STORE.augment( MetaDataStore.DEFAULT_NAME ) ); Files.createFile( storeFile ); } + + private static String formatProperty( Setting setting, Path path ) + { + return format( "%s=%s", setting.name(), path.toString().replace( '\\', '/' ) ); + } } diff --git a/community/dbms/src/test/java/org/neo4j/commandline/dbms/LoadCommandTest.java b/community/dbms/src/test/java/org/neo4j/commandline/dbms/LoadCommandTest.java index 023ee3f4aa7c9..53dd0417d6674 100644 --- a/community/dbms/src/test/java/org/neo4j/commandline/dbms/LoadCommandTest.java +++ b/community/dbms/src/test/java/org/neo4j/commandline/dbms/LoadCommandTest.java @@ -42,6 +42,7 @@ import org.neo4j.commandline.admin.Usage; import org.neo4j.dbms.archive.IncorrectFormat; import org.neo4j.dbms.archive.Loader; +import org.neo4j.graphdb.config.Setting; import org.neo4j.helpers.ArrayUtil; import org.neo4j.io.fs.DefaultFileSystemAbstraction; import org.neo4j.io.fs.FileSystemAbstraction; @@ -63,7 +64,8 @@ import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; -import static org.neo4j.dbms.DatabaseManagementSystemSettings.data_directory; +import static org.neo4j.graphdb.factory.GraphDatabaseSettings.data_directory; +import static org.neo4j.graphdb.factory.GraphDatabaseSettings.logical_logs_location; public class LoadCommandTest { @@ -87,7 +89,7 @@ public void setUp() throws Exception public void shouldLoadTheDatabaseFromTheArchive() throws CommandFailed, IncorrectUsage, IOException, IncorrectFormat { execute( "foo.db" ); - verify( loader ).load( archive, homeDir.resolve( "data/databases/foo.db" ) ); + verify( loader ).load( archive, homeDir.resolve( "data/databases/foo.db" ), homeDir.resolve( "data/databases/foo.db" ) ); } @Test @@ -98,10 +100,24 @@ public void shouldCalculateTheDatabaseDirectoryFromConfig() Path databaseDir = dataDir.resolve( "databases/foo.db" ); Files.createDirectories( databaseDir ); Files.write( configDir.resolve( Config.DEFAULT_CONFIG_FILE_NAME ), - asList( format( "%s=%s", data_directory.name(), dataDir.toString().replace( '\\', '/' ) ) ) ); + asList( formatProperty( data_directory, dataDir ) ) ); execute( "foo.db" ); - verify( loader ).load( any(), eq( databaseDir ) ); + verify( loader ).load( any(), eq( databaseDir ), eq( databaseDir ) ); + } + + @Test + public void shouldCalculateTheTxLogDirectoryFromConfig() throws Exception + { + Path dataDir = testDirectory.directory( "some-other-path" ).toPath(); + Path txLogsDir = testDirectory.directory( "txLogsPath" ).toPath(); + Path databaseDir = dataDir.resolve( "databases/foo.db" ); + Files.write( configDir.resolve( Config.DEFAULT_CONFIG_FILE_NAME ), + asList( formatProperty( data_directory, dataDir ), + formatProperty( logical_logs_location, txLogsDir ) ) ); + + execute( "foo.db" ); + verify( loader ).load( any(), eq( databaseDir ), eq( txLogsDir ) ); } @Test @@ -121,10 +137,10 @@ public void shouldHandleSymlinkToDatabaseDir() throws IOException, CommandFailed Files.createSymbolicLink( databaseDir, realDatabaseDir ); Files.write( configDir.resolve( Config.DEFAULT_CONFIG_FILE_NAME ), - asList( format( "%s=%s", data_directory.name(), dataDir.toString().replace( '\\', '/' ) ) ) ); + asList( formatProperty( data_directory, dataDir ) ) ); execute( "foo.db" ); - verify( loader ).load( any(), eq( realDatabaseDir ) ); + verify( loader ).load( any(), eq( realDatabaseDir ), eq( realDatabaseDir ) ); } @Test @@ -134,12 +150,12 @@ public void shouldMakeFromCanonical() throws IOException, CommandFailed, Incorre Path databaseDir = dataDir.resolve( "databases/foo.db" ); Files.createDirectories( databaseDir ); Files.write( configDir.resolve( Config.DEFAULT_CONFIG_FILE_NAME ), - asList( format( "%s=%s", data_directory.name(), dataDir.toString().replace( '\\', '/' ) ) ) ); + asList( formatProperty( data_directory, dataDir ) ) ); new LoadCommand( homeDir, configDir, loader ) .execute( ArrayUtil.concat( new String[]{"--database=foo.db", "--from=foo.dump"} ) ); - verify( loader ).load( eq( Paths.get( new File( "foo.dump" ).getCanonicalPath() ) ), any() ); + verify( loader ).load( eq( Paths.get( new File( "foo.dump" ).getCanonicalPath() ) ), any(), any() ); } @Test @@ -153,7 +169,7 @@ public void shouldDeleteTheOldDatabaseIfForceArgumentIsProvided() { assertThat( Files.exists( databaseDirectory ), equalTo( false ) ); return null; - } ).when( loader ).load( any(), any() ); + } ).when( loader ).load( any(), any(), any() ); execute( "foo.db", "--force" ); } @@ -169,7 +185,7 @@ public void shouldNotDeleteTheOldDatabaseIfForceArgumentIsNotProvided() { assertThat( Files.exists( databaseDirectory ), equalTo( true ) ); return null; - } ).when( loader ).load( any(), any() ); + } ).when( loader ).load( any(), any(), any() ); execute( "foo.db" ); } @@ -200,7 +216,7 @@ public void shouldDefaultToGraphDb() throws Exception Files.createDirectories( databaseDir ); new LoadCommand( homeDir, configDir, loader ).execute( new String[]{"--from=something"} ); - verify( loader ).load( any(), eq( databaseDir ) ); + verify( loader ).load( any(), eq( databaseDir ), eq( databaseDir ) ); } @Test @@ -220,7 +236,7 @@ public void shouldObjectIfTheArchiveArgumentIsMissing() throws Exception @Test public void shouldGiveAClearMessageIfTheArchiveDoesntExist() throws IOException, IncorrectFormat, IncorrectUsage { - doThrow( new NoSuchFileException( archive.toString() ) ).when( loader ).load( any(), any() ); + doThrow( new NoSuchFileException( archive.toString() ) ).when( loader ).load( any(), any(), any() ); try { execute( null ); @@ -235,7 +251,7 @@ public void shouldGiveAClearMessageIfTheArchiveDoesntExist() throws IOException, @Test public void shouldGiveAClearMessageIfTheDatabaseAlreadyExists() throws IOException, IncorrectFormat, IncorrectUsage { - doThrow( FileAlreadyExistsException.class ).when( loader ).load( any(), any() ); + doThrow( FileAlreadyExistsException.class ).when( loader ).load( any(), any(), any() ); try { execute( "foo.db" ); @@ -251,7 +267,7 @@ public void shouldGiveAClearMessageIfTheDatabaseAlreadyExists() throws IOExcepti public void shouldGiveAClearMessageIfTheDatabasesDirectoryIsNotWritable() throws IOException, IncorrectFormat, IncorrectUsage { - doThrow( AccessDeniedException.class ).when( loader ).load( any(), any() ); + doThrow( AccessDeniedException.class ).when( loader ).load( any(), any(), any() ); try { execute( null ); @@ -265,10 +281,10 @@ public void shouldGiveAClearMessageIfTheDatabasesDirectoryIsNotWritable() } @Test - public void shouldWrapIOExceptionsCarefulllyBecauseCriticalInformationIsOftenEncodedInTheirNameButMissingFromTheirMessage() + public void shouldWrapIOExceptionsCarefullyBecauseCriticalInformationIsOftenEncodedInTheirNameButMissingFromTheirMessage() throws IOException, IncorrectUsage, IncorrectFormat { - doThrow( new FileSystemException( "the-message" ) ).when( loader ).load( any(), any() ); + doThrow( new FileSystemException( "the-message" ) ).when( loader ).load( any(), any(), any() ); try { execute( null ); @@ -283,7 +299,7 @@ public void shouldWrapIOExceptionsCarefulllyBecauseCriticalInformationIsOftenEnc @Test public void shouldThrowIfTheArchiveFormatIsInvalid() throws IOException, IncorrectUsage, IncorrectFormat { - doThrow( IncorrectFormat.class ).when( loader ).load( any(), any() ); + doThrow( IncorrectFormat.class ).when( loader ).load( any(), any(), any() ); try { execute( null ); @@ -335,4 +351,9 @@ private void execute( String database, String... otherArgs ) throws IncorrectUsa new LoadCommand( homeDir, configDir, loader ) .execute( ArrayUtil.concat( new String[]{"--database=" + database, "--from=" + archive}, otherArgs ) ); } + + private static String formatProperty( Setting setting, Path path ) + { + return format( "%s=%s", setting.name(), path.toString().replace( '\\', '/' ) ); + } } diff --git a/community/dbms/src/test/java/org/neo4j/commandline/dbms/UtilTest.java b/community/dbms/src/test/java/org/neo4j/commandline/dbms/UtilTest.java index d38d9e9a5379b..06b6c5e730187 100644 --- a/community/dbms/src/test/java/org/neo4j/commandline/dbms/UtilTest.java +++ b/community/dbms/src/test/java/org/neo4j/commandline/dbms/UtilTest.java @@ -19,15 +19,24 @@ */ package org.neo4j.commandline.dbms; +import org.junit.Rule; import org.junit.Test; import org.neo4j.commandline.Util; +import org.neo4j.test.rule.TestDirectory; +import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; +import static org.neo4j.commandline.Util.isSameOrChildFile; +import static org.neo4j.commandline.Util.isSameOrChildPath; import static org.neo4j.commandline.Util.neo4jVersion; public class UtilTest { + @Rule + public final TestDirectory directory = TestDirectory.testDirectory(); + @Test public void canonicalPath() throws Exception { @@ -39,4 +48,29 @@ public void returnsAVersion() throws Exception { assertNotNull( "A version should be returned", neo4jVersion() ); } + + @Test + public void correctlyIdentifySameOrChildFile() + { + assertTrue( isSameOrChildFile( directory.directory(), directory.directory( "a" ) ) ); + assertTrue( isSameOrChildFile( directory.directory(), directory.directory() ) ); + assertTrue( isSameOrChildFile( directory.directory( "/a/./b" ), directory.directory( "a/b" ) ) ); + assertTrue( isSameOrChildFile( directory.directory( "a/b" ), directory.directory( "/a/./b" ) ) ); + + assertFalse( isSameOrChildFile( directory.directory( "a" ), directory.directory( "b" ) ) ); + } + + @Test + public void correctlyIdentifySameOrChildPath() + { + assertTrue( isSameOrChildPath( directory.directory().toPath(), directory.directory( "a" ).toPath() ) ); + assertTrue( isSameOrChildPath( directory.directory().toPath(), directory.directory().toPath() ) ); + assertTrue( isSameOrChildPath( directory.directory( "/a/./b" ).toPath(), + directory.directory( "a/b" ).toPath() ) ); + assertTrue( isSameOrChildPath( directory.directory( "a/b" ).toPath(), + directory.directory( "/a/./b" ).toPath() ) ); + + assertFalse( isSameOrChildPath( directory.directory( "a" ).toPath(), + directory.directory( "b" ).toPath() ) ); + } } diff --git a/community/dbms/src/test/java/org/neo4j/dbms/DatabaseManagementSystemSettingsTest.java b/community/dbms/src/test/java/org/neo4j/dbms/DatabaseManagementSystemSettingsTest.java index d5b604db222de..56ef58480ca0b 100644 --- a/community/dbms/src/test/java/org/neo4j/dbms/DatabaseManagementSystemSettingsTest.java +++ b/community/dbms/src/test/java/org/neo4j/dbms/DatabaseManagementSystemSettingsTest.java @@ -23,6 +23,7 @@ import java.io.File; +import org.neo4j.graphdb.factory.GraphDatabaseSettings; import org.neo4j.kernel.configuration.Config; import static org.hamcrest.core.IsEqual.equalTo; @@ -33,8 +34,8 @@ public class DatabaseManagementSystemSettingsTest @Test public void shouldPutDatabaseDirectoriesIntoDataDatabases() { - Config config = Config.defaults( DatabaseManagementSystemSettings.data_directory, "the-data-directory" ); - assertThat( config.get( DatabaseManagementSystemSettings.database_path ), + Config config = Config.defaults( GraphDatabaseSettings.data_directory, "the-data-directory" ); + assertThat( config.get( GraphDatabaseSettings.database_path ), equalTo( new File( "the-data-directory/databases/graph.db" ) ) ); } } diff --git a/community/dbms/src/test/java/org/neo4j/dbms/archive/ArchiveTest.java b/community/dbms/src/test/java/org/neo4j/dbms/archive/ArchiveTest.java index bc4d113c08075..2221673f6dcaa 100644 --- a/community/dbms/src/test/java/org/neo4j/dbms/archive/ArchiveTest.java +++ b/community/dbms/src/test/java/org/neo4j/dbms/archive/ArchiveTest.java @@ -30,6 +30,7 @@ import java.util.Map; import org.neo4j.function.Predicates; +import org.neo4j.kernel.impl.transaction.log.files.TransactionLogFiles; import org.neo4j.test.rule.TestDirectory; import static java.nio.file.Files.isDirectory; @@ -121,9 +122,9 @@ public void shouldExcludeFilesMatchedByTheExclusionPredicate() throws IOExceptio Files.write( directory.resolve( "another-file" ), new byte[0] ); Path archive = testDirectory.file( "the-archive.dump" ).toPath(); - new Dumper().dump( directory, archive, path -> path.getFileName().toString().equals( "another-file" ) ); + new Dumper().dump( directory, directory, archive, path -> path.getFileName().toString().equals( "another-file" ) ); Path newDirectory = testDirectory.file( "the-new-directory" ).toPath(); - new Loader().load( archive, newDirectory ); + new Loader().load( archive, newDirectory, newDirectory ); Path expectedOutput = testDirectory.directory( "expected-output" ).toPath(); Files.createDirectories( expectedOutput ); @@ -141,9 +142,9 @@ public void shouldExcludeWholeDirectoriesMatchedByTheExclusionPredicate() throws Files.write( subdir.resolve( "a-file" ), new byte[0] ); Path archive = testDirectory.file( "the-archive.dump" ).toPath(); - new Dumper().dump( directory, archive, path -> path.getFileName().toString().equals( "subdir" ) ); + new Dumper().dump( directory, directory, archive, path -> path.getFileName().toString().equals( "subdir" ) ); Path newDirectory = testDirectory.file( "the-new-directory" ).toPath(); - new Loader().load( archive, newDirectory ); + new Loader().load( archive, newDirectory, newDirectory ); Path expectedOutput = testDirectory.directory( "expected-output" ).toPath(); Files.createDirectories( expectedOutput ); @@ -151,12 +152,38 @@ public void shouldExcludeWholeDirectoriesMatchedByTheExclusionPredicate() throws assertEquals( describeRecursively( expectedOutput ), describeRecursively( newDirectory ) ); } + @Test + public void dumpAndLoadTransactionLogsFromCustomLocations() throws IOException, IncorrectFormat + { + Path directory = testDirectory.directory( "dbDirectory" ).toPath(); + Path txLogsDirectory = testDirectory.directory( "txLogsDirectory" ).toPath(); + Files.write( directory.resolve( "dbfile" ), new byte[0] ); + Files.write( txLogsDirectory.resolve( TransactionLogFiles.DEFAULT_NAME + ".0" ), new byte[0] ); + + Path archive = testDirectory.file( "the-archive.dump" ).toPath(); + new Dumper().dump( directory, txLogsDirectory, archive, Predicates.alwaysFalse() ); + Path newDirectory = testDirectory.file( "the-new-directory" ).toPath(); + Path newTxLogsDirectory = testDirectory.file( "newTxLogsDirectory" ).toPath(); + new Loader().load( archive, newDirectory, newTxLogsDirectory ); + + Path expectedOutput = testDirectory.directory( "expected-output" ).toPath(); + Files.createDirectories( expectedOutput ); + Files.write( expectedOutput.resolve( "dbfile" ), new byte[0] ); + + Path expectedTxLogs = testDirectory.directory( "expectedTxLogs" ).toPath(); + Files.createDirectories( expectedTxLogs ); + Files.write( expectedTxLogs.resolve( TransactionLogFiles.DEFAULT_NAME + ".0" ), new byte[0] ); + + assertEquals( describeRecursively( expectedOutput ), describeRecursively( newDirectory ) ); + assertEquals( describeRecursively( expectedTxLogs ), describeRecursively( newTxLogsDirectory ) ); + } + private void assertRoundTrips( Path oldDirectory ) throws IOException, IncorrectFormat { Path archive = testDirectory.file( "the-archive.dump" ).toPath(); - new Dumper().dump( oldDirectory, archive, Predicates.alwaysFalse() ); + new Dumper().dump( oldDirectory, oldDirectory, archive, Predicates.alwaysFalse() ); Path newDirectory = testDirectory.file( "the-new-directory" ).toPath(); - new Loader().load( archive, newDirectory ); + new Loader().load( archive, newDirectory, newDirectory ); assertEquals( describeRecursively( oldDirectory ), describeRecursively( newDirectory ) ); } diff --git a/community/dbms/src/test/java/org/neo4j/dbms/archive/DumperTest.java b/community/dbms/src/test/java/org/neo4j/dbms/archive/DumperTest.java index 734e8a90a2406..834654638f945 100644 --- a/community/dbms/src/test/java/org/neo4j/dbms/archive/DumperTest.java +++ b/community/dbms/src/test/java/org/neo4j/dbms/archive/DumperTest.java @@ -19,6 +19,10 @@ */ package org.neo4j.dbms.archive; +import org.apache.commons.lang3.SystemUtils; +import org.junit.Rule; +import org.junit.Test; + import java.io.Closeable; import java.io.IOException; import java.nio.file.AccessDeniedException; @@ -28,15 +32,10 @@ import java.nio.file.NoSuchFileException; import java.nio.file.Path; -import org.apache.commons.lang3.SystemUtils; -import org.junit.Rule; -import org.junit.Test; - import org.neo4j.function.Predicates; import org.neo4j.test.rule.TestDirectory; import static java.util.Collections.emptySet; - import static org.junit.Assert.assertEquals; import static org.junit.Assert.fail; import static org.junit.Assume.assumeFalse; @@ -54,7 +53,7 @@ public void shouldGiveAClearErrorIfTheArchiveAlreadyExists() throws IOException Files.write( archive, new byte[0] ); try { - new Dumper().dump( directory, archive, Predicates.alwaysFalse() ); + new Dumper().dump( directory, directory, archive, Predicates.alwaysFalse() ); fail( "Expected an exception" ); } catch ( FileAlreadyExistsException e ) @@ -70,7 +69,7 @@ public void shouldGiveAClearErrorMessageIfTheDirectoryDoesntExist() throws IOExc Path archive = testDirectory.file( "the-archive.dump" ).toPath(); try { - new Dumper().dump( directory, archive, Predicates.alwaysFalse() ); + new Dumper().dump( directory, directory, archive, Predicates.alwaysFalse() ); fail( "Expected an exception" ); } catch ( NoSuchFileException e ) @@ -86,7 +85,7 @@ public void shouldGiveAClearErrorMessageIfTheArchivesParentDirectoryDoesntExist( Path archive = testDirectory.file( "subdir/the-archive.dump" ).toPath(); try { - new Dumper().dump( directory, archive, Predicates.alwaysFalse() ); + new Dumper().dump( directory, directory, archive, Predicates.alwaysFalse() ); fail( "Expected an exception" ); } catch ( NoSuchFileException e ) @@ -103,7 +102,7 @@ public void shouldGiveAClearErrorMessageIfTheArchivesParentDirectoryIsAFile() th Files.write( archive.getParent(), new byte[0] ); try { - new Dumper().dump( directory, archive, Predicates.alwaysFalse() ); + new Dumper().dump( directory, directory, archive, Predicates.alwaysFalse() ); fail( "Expected an exception" ); } catch ( FileSystemException e ) @@ -122,7 +121,7 @@ public void shouldGiveAClearErrorMessageIfTheArchivesParentDirectoryIsNotWritabl Files.createDirectories( archive.getParent() ); try ( Closeable ignored = TestUtils.withPermissions( archive.getParent(), emptySet() ) ) { - new Dumper().dump( directory, archive, Predicates.alwaysFalse() ); + new Dumper().dump( directory, directory, archive, Predicates.alwaysFalse() ); fail( "Expected an exception" ); } catch ( AccessDeniedException e ) diff --git a/community/dbms/src/test/java/org/neo4j/dbms/archive/LoaderTest.java b/community/dbms/src/test/java/org/neo4j/dbms/archive/LoaderTest.java index 7afd0298dcc62..0521d28b40313 100644 --- a/community/dbms/src/test/java/org/neo4j/dbms/archive/LoaderTest.java +++ b/community/dbms/src/test/java/org/neo4j/dbms/archive/LoaderTest.java @@ -19,6 +19,11 @@ */ package org.neo4j.dbms.archive; +import org.apache.commons.compress.compressors.gzip.GzipCompressorOutputStream; +import org.apache.commons.lang3.SystemUtils; +import org.junit.Rule; +import org.junit.Test; + import java.io.Closeable; import java.io.IOException; import java.nio.file.AccessDeniedException; @@ -29,20 +34,13 @@ import java.nio.file.Path; import java.util.Random; -import org.apache.commons.compress.compressors.gzip.GzipCompressorOutputStream; -import org.apache.commons.lang3.SystemUtils; -import org.junit.Rule; -import org.junit.Test; - import org.neo4j.test.rule.TestDirectory; import static java.util.Arrays.asList; import static java.util.Collections.emptySet; - import static org.junit.Assert.assertEquals; import static org.junit.Assert.fail; import static org.junit.Assume.assumeFalse; - import static org.neo4j.dbms.archive.TestUtils.withPermissions; public class LoaderTest @@ -57,7 +55,7 @@ public void shouldGiveAClearErrorMessageIfTheArchiveDoesntExist() throws IOExcep Path destination = testDirectory.file( "the-destination" ).toPath(); try { - new Loader().load( archive, destination ); + new Loader().load( archive, destination, destination ); fail( "Expected an exception" ); } catch ( NoSuchFileException e ) @@ -74,7 +72,7 @@ public void shouldGiveAClearErrorMessageIfTheArchiveIsNotInGzipFormat() throws I Path destination = testDirectory.file( "the-destination" ).toPath(); try { - new Loader().load( archive, destination ); + new Loader().load( archive, destination, destination ); fail( "Expected an exception" ); } catch ( IncorrectFormat e ) @@ -98,7 +96,7 @@ public void shouldGiveAClearErrorMessageIfTheArchiveIsNotInTarFormat() throws IO Path destination = testDirectory.file( "the-destination" ).toPath(); try { - new Loader().load( archive, destination ); + new Loader().load( archive, destination, destination ); fail( "Expected an exception" ); } catch ( IncorrectFormat e ) @@ -114,7 +112,7 @@ public void shouldGiveAClearErrorIfTheDestinationAlreadyExists() throws IOExcept Path destination = testDirectory.directory( "the-destination" ).toPath(); try { - new Loader().load( archive, destination ); + new Loader().load( archive, destination, destination ); fail( "Expected an exception" ); } catch ( FileAlreadyExistsException e ) @@ -123,6 +121,23 @@ public void shouldGiveAClearErrorIfTheDestinationAlreadyExists() throws IOExcept } } + @Test + public void shouldGiveAClearErrorIfTheDestinationTxLogAlreadyExists() throws IOException, IncorrectFormat + { + Path archive = testDirectory.file( "the-archive.dump" ).toPath(); + Path destination = testDirectory.file( "the-destination" ).toPath(); + Path txLogsDestination = testDirectory.directory( "txLogsDestination" ).toPath(); + try + { + new Loader().load( archive, destination, txLogsDestination ); + fail( "Expected an exception" ); + } + catch ( FileAlreadyExistsException e ) + { + assertEquals( txLogsDestination.toString(), e.getMessage() ); + } + } + @Test public void shouldGiveAClearErrorMessageIfTheDestinationsParentDirectoryDoesntExist() throws IOException, IncorrectFormat @@ -131,7 +146,7 @@ public void shouldGiveAClearErrorMessageIfTheDestinationsParentDirectoryDoesntEx Path destination = testDirectory.directory( "subdir/the-destination" ).toPath(); try { - new Loader().load( archive, destination ); + new Loader().load( archive, destination, destination ); fail( "Expected an exception" ); } catch ( NoSuchFileException e ) @@ -140,6 +155,24 @@ public void shouldGiveAClearErrorMessageIfTheDestinationsParentDirectoryDoesntEx } } + @Test + public void shouldGiveAClearErrorMessageIfTheTxLogsParentDirectoryDoesntExist() + throws IOException, IncorrectFormat + { + Path archive = testDirectory.file( "the-archive.dump" ).toPath(); + Path destination = testDirectory.file( "destination" ).toPath(); + Path txLogsDestination = testDirectory.directory( "subdir/txLogs" ).toPath(); + try + { + new Loader().load( archive, destination, txLogsDestination ); + fail( "Expected an exception" ); + } + catch ( NoSuchFileException e ) + { + assertEquals( txLogsDestination.getParent().toString(), e.getMessage() ); + } + } + @Test public void shouldGiveAClearErrorMessageIfTheDestinationsParentDirectoryIsAFile() throws IOException, IncorrectFormat @@ -149,7 +182,7 @@ public void shouldGiveAClearErrorMessageIfTheDestinationsParentDirectoryIsAFile( Files.write( destination.getParent(), new byte[0] ); try { - new Loader().load( archive, destination ); + new Loader().load( archive, destination, destination ); fail( "Expected an exception" ); } catch ( FileSystemException e ) @@ -169,7 +202,7 @@ public void shouldGiveAClearErrorMessageIfTheDestinationsParentDirectoryIsNotWri Files.createDirectories( destination.getParent() ); try ( Closeable ignored = withPermissions( destination.getParent(), emptySet() ) ) { - new Loader().load( archive, destination ); + new Loader().load( archive, destination, destination ); fail( "Expected an exception" ); } catch ( AccessDeniedException e ) @@ -177,4 +210,25 @@ public void shouldGiveAClearErrorMessageIfTheDestinationsParentDirectoryIsNotWri assertEquals( destination.getParent().toString(), e.getMessage() ); } } + + @Test + public void shouldGiveAClearErrorMessageIfTheTxLogsParentDirectoryIsNotWritable() + throws IOException, IncorrectFormat + { + assumeFalse( "We haven't found a way to reliably tests permissions on Windows", SystemUtils.IS_OS_WINDOWS ); + + Path archive = testDirectory.file( "the-archive.dump" ).toPath(); + Path destination = testDirectory.file( "destination" ).toPath(); + Path txLogsDrectory = testDirectory.directory( "subdir/txLogs" ).toPath(); + Files.createDirectories( txLogsDrectory.getParent() ); + try ( Closeable ignored = withPermissions( txLogsDrectory.getParent(), emptySet() ) ) + { + new Loader().load( archive, destination, txLogsDrectory ); + fail( "Expected an exception" ); + } + catch ( AccessDeniedException e ) + { + assertEquals( txLogsDrectory.getParent().toString(), e.getMessage() ); + } + } } diff --git a/community/io/src/main/java/org/neo4j/io/fs/DefaultFileSystemAbstraction.java b/community/io/src/main/java/org/neo4j/io/fs/DefaultFileSystemAbstraction.java index 36e866ede85d3..fcd156d27a6e1 100644 --- a/community/io/src/main/java/org/neo4j/io/fs/DefaultFileSystemAbstraction.java +++ b/community/io/src/main/java/org/neo4j/io/fs/DefaultFileSystemAbstraction.java @@ -178,6 +178,12 @@ public void moveToDirectory( File file, File toDirectory ) throws IOException FileUtils.moveFileToDirectory( file, toDirectory ); } + @Override + public void copyToDirectory( File file, File toDirectory ) throws IOException + { + FileUtils.copyFileToDirectory( file, toDirectory ); + } + @Override public void copyFile( File from, File to ) throws IOException { diff --git a/community/io/src/main/java/org/neo4j/io/fs/DelegateFileSystemAbstraction.java b/community/io/src/main/java/org/neo4j/io/fs/DelegateFileSystemAbstraction.java index b7221027d719c..518673c63e235 100644 --- a/community/io/src/main/java/org/neo4j/io/fs/DelegateFileSystemAbstraction.java +++ b/community/io/src/main/java/org/neo4j/io/fs/DelegateFileSystemAbstraction.java @@ -227,6 +227,12 @@ public void moveToDirectory( File file, File toDirectory ) throws IOException Files.move( path( file ), path( toDirectory ).resolve( path( file.getName() ) ) ); } + @Override + public void copyToDirectory( File file, File toDirectory ) throws IOException + { + Files.copy( path( file ), toDirectory.toPath().resolve( file.getName() ) ); + } + @Override public void copyFile( File from, File to ) throws IOException { diff --git a/community/io/src/main/java/org/neo4j/io/fs/FileSystemAbstraction.java b/community/io/src/main/java/org/neo4j/io/fs/FileSystemAbstraction.java index 03966217f08ff..22715493418ba 100644 --- a/community/io/src/main/java/org/neo4j/io/fs/FileSystemAbstraction.java +++ b/community/io/src/main/java/org/neo4j/io/fs/FileSystemAbstraction.java @@ -82,6 +82,8 @@ public interface FileSystemAbstraction extends Closeable void moveToDirectory( File file, File toDirectory ) throws IOException; + void copyToDirectory( File file, File toDirectory ) throws IOException; + void copyFile( File from, File to ) throws IOException; void copyRecursively( File fromDirectory, File toDirectory ) throws IOException; diff --git a/community/io/src/main/java/org/neo4j/io/fs/FileUtils.java b/community/io/src/main/java/org/neo4j/io/fs/FileUtils.java index 2ec14011b416d..c4e0113f07728 100644 --- a/community/io/src/main/java/org/neo4j/io/fs/FileUtils.java +++ b/community/io/src/main/java/org/neo4j/io/fs/FileUtils.java @@ -51,6 +51,7 @@ import java.nio.file.Path; import java.nio.file.Paths; import java.nio.file.SimpleFileVisitor; +import java.nio.file.StandardCopyOption; import java.nio.file.StandardOpenOption; import java.nio.file.attribute.BasicFileAttributes; @@ -190,6 +191,30 @@ public static File moveFileToDirectory( File toMove, File targetDirectory ) thro return target; } + /** + * Utility method that copy a file from its current location to the + * provided target directory. + * + * @param file file that needs to be copied. + * @param targetDirectory the destination directory + * @throws IOException + */ + public static void copyFileToDirectory( File file, File targetDirectory ) throws IOException + { + if ( !targetDirectory.exists() ) + { + Files.createDirectories( targetDirectory.toPath() ); + } + if ( !targetDirectory.isDirectory() ) + { + throw new IllegalArgumentException( + "Move target must be a directory, not " + targetDirectory ); + } + + File target = new File( targetDirectory, file.getName() ); + copyFile( file, target ); + } + public static void renameFile( File srcFile, File renameToFile, CopyOption... copyOptions ) throws IOException { Files.move( srcFile.toPath(), renameToFile.toPath(), copyOptions ); @@ -264,22 +289,7 @@ public static void copyFile( File srcFile, File dstFile ) throws IOException { //noinspection ResultOfMethodCallIgnored dstFile.getParentFile().mkdirs(); - try ( FileInputStream input = new FileInputStream( srcFile ); - FileOutputStream output = new FileOutputStream( dstFile ) ) - { - int bufferSize = 1024; - byte[] buffer = new byte[bufferSize]; - int bytesRead; - while ( (bytesRead = input.read( buffer )) != -1 ) - { - output.write( buffer, 0, bytesRead ); - } - } - catch ( IOException e ) - { - // Because the message from this cause may not mention which file it's about - throw new IOException( "Could not copy '" + srcFile + "' to '" + dstFile + "'", e ); - } + Files.copy( srcFile.toPath(), dstFile.toPath(), StandardCopyOption.REPLACE_EXISTING ); } public static void copyRecursively( File fromDirectory, File toDirectory ) throws IOException diff --git a/community/io/src/test/java/org/neo4j/adversaries/fs/AdversarialFileSystemAbstraction.java b/community/io/src/test/java/org/neo4j/adversaries/fs/AdversarialFileSystemAbstraction.java index bc02ab2766476..8f5a07a45c585 100644 --- a/community/io/src/test/java/org/neo4j/adversaries/fs/AdversarialFileSystemAbstraction.java +++ b/community/io/src/test/java/org/neo4j/adversaries/fs/AdversarialFileSystemAbstraction.java @@ -185,6 +185,15 @@ public void moveToDirectory( File file, File toDirectory ) throws IOException delegate.moveToDirectory( file, toDirectory ); } + @Override + public void copyToDirectory( File file, File toDirectory ) throws IOException + { + adversary.injectFailure( + SecurityException.class, IllegalArgumentException.class, FileNotFoundException.class, + NullPointerException.class, IOException.class ); + delegate.copyToDirectory( file, toDirectory ); + } + @Override public boolean isDirectory( File file ) { diff --git a/community/io/src/test/java/org/neo4j/graphdb/mockfs/DelegatingFileSystemAbstraction.java b/community/io/src/test/java/org/neo4j/graphdb/mockfs/DelegatingFileSystemAbstraction.java index 12da53dbaa1f8..176855974b3b7 100644 --- a/community/io/src/test/java/org/neo4j/graphdb/mockfs/DelegatingFileSystemAbstraction.java +++ b/community/io/src/test/java/org/neo4j/graphdb/mockfs/DelegatingFileSystemAbstraction.java @@ -65,6 +65,12 @@ public void moveToDirectory( File file, File toDirectory ) throws IOException delegate.moveToDirectory( file, toDirectory ); } + @Override + public void copyToDirectory( File file, File toDirectory ) throws IOException + { + delegate.copyToDirectory( file, toDirectory ); + } + @Override public boolean mkdir( File fileName ) { diff --git a/community/io/src/test/java/org/neo4j/graphdb/mockfs/EphemeralFileSystemAbstraction.java b/community/io/src/test/java/org/neo4j/graphdb/mockfs/EphemeralFileSystemAbstraction.java index 8d1a216df3a7f..447ff5f94e914 100644 --- a/community/io/src/test/java/org/neo4j/graphdb/mockfs/EphemeralFileSystemAbstraction.java +++ b/community/io/src/test/java/org/neo4j/graphdb/mockfs/EphemeralFileSystemAbstraction.java @@ -540,6 +540,13 @@ public void moveToDirectory( File file, File toDirectory ) throws IOException } } + @Override + public void copyToDirectory( File file, File toDirectory ) throws IOException + { + File targetFile = new File( toDirectory, file.getName() ); + copyFile( file, targetFile ); + } + @Override public void copyFile( File from, File to ) throws IOException { @@ -626,6 +633,7 @@ private void copyFile( File from, FileSystemAbstraction fromFs, File to, ByteBuf try ( StoreChannel source = fromFs.open( from, OpenMode.READ ); StoreChannel sink = this.open( to, OpenMode.READ_WRITE ) ) { + sink.truncate( 0 ); for ( int available; (available = (int) (source.size() - source.position())) > 0; ) { buffer.clear(); diff --git a/community/io/src/test/java/org/neo4j/graphdb/mockfs/SelectiveFileSystemAbstraction.java b/community/io/src/test/java/org/neo4j/graphdb/mockfs/SelectiveFileSystemAbstraction.java index bdf90c0b1a6cf..1155f96a1a806 100644 --- a/community/io/src/test/java/org/neo4j/graphdb/mockfs/SelectiveFileSystemAbstraction.java +++ b/community/io/src/test/java/org/neo4j/graphdb/mockfs/SelectiveFileSystemAbstraction.java @@ -167,6 +167,12 @@ public void moveToDirectory( File file, File toDirectory ) throws IOException chooseFileSystem( file ).moveToDirectory( file, toDirectory ); } + @Override + public void copyToDirectory( File file, File toDirectory ) throws IOException + { + chooseFileSystem( file ).copyToDirectory( file, toDirectory ); + } + @Override public void copyFile( File from, File to ) throws IOException { diff --git a/community/io/src/test/java/org/neo4j/io/fs/FileSystemAbstractionTest.java b/community/io/src/test/java/org/neo4j/io/fs/FileSystemAbstractionTest.java index b39d0feccd17e..ad96f2ec392d7 100644 --- a/community/io/src/test/java/org/neo4j/io/fs/FileSystemAbstractionTest.java +++ b/community/io/src/test/java/org/neo4j/io/fs/FileSystemAbstractionTest.java @@ -55,6 +55,7 @@ import static org.hamcrest.Matchers.not; import static org.hamcrest.Matchers.nullValue; import static org.hamcrest.core.Is.is; +import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertSame; @@ -158,25 +159,39 @@ public void moveToDirectoryMustMoveFile() throws Exception } @Test - public void moveToDirectoryMustRecursivelyMoveAllFilesInGivenDirectory() throws Exception + public void copyToDirectoryCopiesFile() throws IOException { File source = new File( path, "source" ); File target = new File( path, "target" ); File file = new File( source, "file" ); - File sourceAfterMove = new File( target, "source" ); - File fileAfterMove = new File( sourceAfterMove, "file" ); + File fileAfterCopy = new File( target, "file" ); fsa.mkdirs( source ); fsa.mkdirs( target ); fsa.create( file ).close(); - assertTrue( fsa.fileExists( source ) ); assertTrue( fsa.fileExists( file ) ); - assertFalse( fsa.fileExists( sourceAfterMove ) ); - assertFalse( fsa.fileExists( fileAfterMove ) ); - fsa.moveToDirectory( source, target ); - assertFalse( fsa.fileExists( source ) ); - assertFalse( fsa.fileExists( file ) ); - assertTrue( fsa.fileExists( sourceAfterMove ) ); - assertTrue( fsa.fileExists( fileAfterMove ) ); + assertFalse( fsa.fileExists( fileAfterCopy ) ); + fsa.copyToDirectory( file, target ); + assertTrue( fsa.fileExists( file ) ); + assertTrue( fsa.fileExists( fileAfterCopy ) ); + } + + @Test + public void copyToDirectoryReplaceExistingFile() throws Exception + { + File source = new File( path, "source" ); + File target = new File( path, "target" ); + File file = new File( source, "file" ); + File targetFile = new File( target, "file" ); + fsa.mkdirs( source ); + fsa.mkdirs( target ); + fsa.create( file ).close(); + + writeIntegerIntoFile( targetFile ); + + fsa.copyToDirectory( file, target ); + assertTrue( fsa.fileExists( file ) ); + assertTrue( fsa.fileExists( targetFile ) ); + assertEquals( 0L, fsa.getFileSize( targetFile ) ); } @Test @@ -812,4 +827,13 @@ private void ensureDirectoryExists( File directory ) throws IOException { fsa.mkdirs( directory ); } + + private void writeIntegerIntoFile( File targetFile ) throws IOException + { + StoreChannel storeChannel = fsa.create( targetFile ); + ByteBuffer byteBuffer = ByteBuffer.allocate( Integer.SIZE ).putInt( 7 ); + byteBuffer.flip(); + storeChannel.writeAll( byteBuffer ); + storeChannel.close(); + } } diff --git a/community/io/src/test/java/org/neo4j/test/rule/fs/FileSystemRule.java b/community/io/src/test/java/org/neo4j/test/rule/fs/FileSystemRule.java index 9aeac9f09f3de..680f473f3c241 100644 --- a/community/io/src/test/java/org/neo4j/test/rule/fs/FileSystemRule.java +++ b/community/io/src/test/java/org/neo4j/test/rule/fs/FileSystemRule.java @@ -185,6 +185,12 @@ public void moveToDirectory( File file, File toDirectory ) throws IOException fs.moveToDirectory( file, toDirectory ); } + @Override + public void copyToDirectory( File file, File toDirectory ) throws IOException + { + fs.copyToDirectory( file, toDirectory ); + } + @Override public void copyFile( File from, File to ) throws IOException { diff --git a/community/kernel/src/main/java/org/neo4j/graphdb/factory/GraphDatabaseSettings.java b/community/kernel/src/main/java/org/neo4j/graphdb/factory/GraphDatabaseSettings.java index c11ed027dc02d..a1260a21d34a4 100644 --- a/community/kernel/src/main/java/org/neo4j/graphdb/factory/GraphDatabaseSettings.java +++ b/community/kernel/src/main/java/org/neo4j/graphdb/factory/GraphDatabaseSettings.java @@ -101,6 +101,19 @@ public class GraphDatabaseSettings implements LoadableConfig public static final Setting neo4j_home = setting( "unsupported.dbms.directories.neo4j_home", PATH, NO_DEFAULT ); + @Description( "Name of the database to load" ) + public static final Setting active_database = setting( "dbms.active_database", STRING, "graph.db" ); + + @Description( "Path of the data directory. You must not configure more than one Neo4j installation to use the " + + "same data directory." ) + public static final Setting data_directory = pathSetting( "dbms.directories.data", "data" ); + + @Internal + public static final Setting database_path = derivedSetting( "unsupported.dbms.directories.database", + data_directory, active_database, + ( data, current ) -> new File( new File( data, "databases" ), current ), + PATH ); + @Title( "Read only database" ) @Description( "Only allow read operations from this Neo4j instance. " + "This mode still requires write access to the directory for lock purposes." ) @@ -433,6 +446,10 @@ public class GraphDatabaseSettings implements LoadableConfig public static final Setting enable_native_schema_index = setting( "unsupported.dbms.enable_native_schema_index", BOOLEAN, TRUE ); + @Description( "Location where Neo4j keeps the logical transaction logs." ) + public static final Setting logical_logs_location = + pathSetting( "dbms.directories.tx_log", "", database_path ); + // Store settings @Description( "Make Neo4j keep the logical transaction logs for being able to backup the database. " + "Can be used for specifying the threshold to prune logical logs after. For example \"10 days\" will " + diff --git a/community/kernel/src/main/java/org/neo4j/kernel/NeoStoreDataSource.java b/community/kernel/src/main/java/org/neo4j/kernel/NeoStoreDataSource.java index bc73d9c7254ae..3a8fb5d2f8e8c 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/NeoStoreDataSource.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/NeoStoreDataSource.java @@ -401,10 +401,10 @@ public void start() throws IOException final LogEntryReader logEntryReader = new VersionAwareLogEntryReader<>(); LogFiles logFiles = LogFilesBuilder.builder(storeDir, fs) - .withLogEntryReader( logEntryReader ) - .withLogFileMonitor( physicalLogMonitor ) - .withConfig( config ) - .withDependencies( dependencies ).build(); + .withLogEntryReader( logEntryReader ) + .withLogFileMonitor( physicalLogMonitor ) + .withConfig( config ) + .withDependencies( dependencies ).build(); LogTailScanner tailScanner = new LogTailScanner( logFiles, logEntryReader, monitors, failOnCorruptedLogFiles ); monitors.addMonitorListener( new LoggingLogTailScannerMonitor( logService.getInternalLog( LogTailScanner.class ) ) ); @@ -450,6 +450,7 @@ public void start() throws IOException PropertyAccessor propertyAccessor = dependencies.resolveDependency( PropertyAccessor.class ); final NeoStoreKernelModule kernelModule = buildKernel( + logFiles, transactionLogModule.transactionAppender(), dependencies.resolveDependency( IndexingService.class ), storageEngine.storeReadLayer(), @@ -647,16 +648,10 @@ private void buildRecovery( life.add( recovery ); } - private NeoStoreKernelModule buildKernel( TransactionAppender appender, - IndexingService indexingService, - StoreReadLayer storeLayer, - DatabaseSchemaState databaseSchemaState, LabelScanStore labelScanStore, - StorageEngine storageEngine, - IndexConfigStore indexConfigStore, - TransactionIdStore transactionIdStore, - AvailabilityGuard availabilityGuard, - SystemNanoClock clock, - PropertyAccessor propertyAccessor ) throws KernelException, IOException + private NeoStoreKernelModule buildKernel( LogFiles logFiles, TransactionAppender appender, IndexingService indexingService, + StoreReadLayer storeLayer, DatabaseSchemaState databaseSchemaState, LabelScanStore labelScanStore, + StorageEngine storageEngine, IndexConfigStore indexConfigStore, TransactionIdStore transactionIdStore, + AvailabilityGuard availabilityGuard, SystemNanoClock clock, PropertyAccessor propertyAccessor ) throws KernelException, IOException { CpuClock cpuClock = CpuClock.NOT_AVAILABLE; if ( config.get( GraphDatabaseSettings.track_query_cpu_time ) ) @@ -702,8 +697,8 @@ private NeoStoreKernelModule buildKernel( TransactionAppender appender, kernel.registerTransactionHook( transactionEventHandlers ); - final NeoStoreFileListing fileListing = new NeoStoreFileListing( storeDir, labelScanStore, indexingService, - explicitIndexProviderLookup, storageEngine ); + final NeoStoreFileListing fileListing = new NeoStoreFileListing( storeDir, logFiles, labelScanStore, + indexingService, explicitIndexProviderLookup, storageEngine ); return new NeoStoreKernelModule( transactionCommitProcess, kernel, kernelTransactions, fileListing ); } diff --git a/community/kernel/src/main/java/org/neo4j/kernel/configuration/Settings.java b/community/kernel/src/main/java/org/neo4j/kernel/configuration/Settings.java index 869e3baa18a25..309e8ad66f3c4 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/configuration/Settings.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/configuration/Settings.java @@ -55,6 +55,7 @@ import static java.lang.Float.parseFloat; import static java.lang.Integer.parseInt; import static java.lang.Long.parseLong; +import static java.lang.String.format; import static org.neo4j.graphdb.factory.GraphDatabaseSettings.default_advertised_address; import static org.neo4j.graphdb.factory.GraphDatabaseSettings.default_listen_address; import static org.neo4j.io.fs.FileUtils.fixSeparatorsInPath; @@ -332,58 +333,12 @@ public String valueDescription() public static Setting pathSetting( String name, String defaultValue ) { - return new ScopeAwareSetting() - { - @Override - protected String provideName() - { - return name; - } - - @Override - public String getDefaultValue() - { - return defaultValue; - } - - @Override - public File from( Configuration config ) - { - return config.get( this ); - } - - @Override - public File apply( Function config ) - { - String value = config.apply( name() ); - if ( value == null ) - { - value = defaultValue; - } - if ( value == null ) - { - return null; - } - - String setting = fixSeparatorsInPath( value ); - File settingFile = new File( setting ); - - if ( settingFile.isAbsolute() ) - { - return settingFile; - } - else - { - return new File( GraphDatabaseSettings.neo4j_home.apply( config ), setting ); - } - } + return new FileSetting( name, defaultValue ); + } - @Override - public String valueDescription() - { - return "A filesystem path; relative paths are resolved against the installation root, __"; - } - }; + public static Setting pathSetting( String name, String defaultValue, Setting relativeRoot ) + { + return new FileSetting( name, defaultValue, relativeRoot ); } private static BiFunction, String> inheritedValue( @@ -734,7 +689,7 @@ else if ( mem.endsWith( "g" ) ) } catch ( NumberFormatException e ) { - throw new IllegalArgumentException( String.format( "%s is not a valid size, must be e.g. 10, 5K, 1M, " + + throw new IllegalArgumentException( format( "%s is not a valid size, must be e.g. 10, 5K, 1M, " + "11G", value ) ); } } @@ -935,7 +890,7 @@ public String apply( String value, Function settings ) @Override public String toString() { - return String.format( MATCHES_PATTERN_MESSAGE, regex ); + return format( MATCHES_PATTERN_MESSAGE, regex ); } }; } @@ -983,7 +938,7 @@ public List apply( List values, Function settings @Override public String toString() { - return String.format( MATCHES_PATTERN_MESSAGE, regex ); + return format( MATCHES_PATTERN_MESSAGE, regex ); } }; } @@ -997,7 +952,7 @@ public T apply( T value, Function settings ) { if ( value != null && value.compareTo( min ) < 0 ) { - throw new IllegalArgumentException( String.format( "minimum allowed value is: %s", min ) ); + throw new IllegalArgumentException( format( "minimum allowed value is: %s", min ) ); } return value; } @@ -1019,7 +974,7 @@ public T apply( T value, Function settings ) { if ( value != null && value.compareTo( max ) > 0 ) { - throw new IllegalArgumentException( String.format( "maximum allowed value is: %s", max ) ); + throw new IllegalArgumentException( format( "maximum allowed value is: %s", max ) ); } return value; } @@ -1045,7 +1000,7 @@ public T apply( T from1, Function from2 ) @Override public String toString() { - return String.format( "is in the range `%s` to `%s`", min, max ); + return format( "is in the range `%s` to `%s`", min, max ); } }; } @@ -1076,7 +1031,7 @@ public String toString() { String description = message; if ( valueFunction != null - && !String.format( MATCHES_PATTERN_MESSAGE, ANY ).equals( + && !format( MATCHES_PATTERN_MESSAGE, ANY ).equals( valueFunction.toString() ) ) { description += " (" + valueFunction.toString() + ")"; @@ -1311,7 +1266,7 @@ public T apply( Function settings ) } catch ( Exception e ) { - throw new IllegalArgumentException( String.format( "Missing mandatory setting '%s'", name() ) ); + throw new IllegalArgumentException( format( "Missing mandatory setting '%s'", name() ) ); } } @@ -1367,4 +1322,73 @@ public String valueDescription() return builder.toString(); } } + + private static class FileSetting extends ScopeAwareSetting + { + private final String name; + private final String defaultValue; + private final Setting relativeRoot; + + FileSetting( String name, String defaultValue ) + { + this( name, defaultValue, GraphDatabaseSettings.neo4j_home ); + } + + FileSetting( String name, String defaultValue, Setting relativeRoot ) + { + this.name = name; + this.defaultValue = defaultValue; + this.relativeRoot = relativeRoot; + } + + @Override + protected String provideName() + { + return name; + } + + @Override + public String getDefaultValue() + { + return defaultValue; + } + + @Override + public File from( Configuration config ) + { + return config.get( this ); + } + + @Override + public File apply( Function config ) + { + String value = config.apply( name() ); + if ( value == null ) + { + value = defaultValue; + } + if ( value == null ) + { + return null; + } + + String setting = fixSeparatorsInPath( value ); + File settingFile = new File( setting ); + + if ( settingFile.isAbsolute() ) + { + return settingFile; + } + else + { + return new File( relativeRoot.apply( config ), setting ); + } + } + + @Override + public String valueDescription() + { + return "A filesystem path; relative paths are resolved against the root, _<" + relativeRoot.name() + ">_"; + } + } } diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/factory/PlatformModule.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/factory/PlatformModule.java index 4cf27a98c0e18..af15ad0436ef4 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/factory/PlatformModule.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/factory/PlatformModule.java @@ -179,10 +179,6 @@ public PlatformModule( File providedStoreDir, Config config, DatabaseInfo databa diagnosticsManager = life.add( dependencies .satisfyDependency( new DiagnosticsManager( logging.getInternalLog( DiagnosticsManager.class ) ) ) ); - // TODO please fix the bad dependencies instead of doing this. - // this was the place of the XaDataSourceManager. NeoStoreXaDataSource is create further down than - // (specifically) KernelExtensions, which creates an interesting out-of-order issue with #doAfterRecovery(). - // Anyways please fix this. dependencies.satisfyDependency( dataSourceManager ); availabilityGuard = dependencies.satisfyDependency( createAvailabilityGuard() ); diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/recovery/RecoveryRequiredChecker.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/recovery/RecoveryRequiredChecker.java index b398bbf67f227..a14fc8728fe47 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/recovery/RecoveryRequiredChecker.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/recovery/RecoveryRequiredChecker.java @@ -24,6 +24,7 @@ import org.neo4j.io.fs.FileSystemAbstraction; import org.neo4j.io.pagecache.PageCache; +import org.neo4j.kernel.configuration.Config; import org.neo4j.kernel.impl.store.NeoStores; import org.neo4j.kernel.impl.transaction.log.ReadableClosablePositionAwareChannel; import org.neo4j.kernel.impl.transaction.log.entry.LogEntryReader; @@ -44,11 +45,13 @@ public class RecoveryRequiredChecker private final FileSystemAbstraction fs; private final PageCache pageCache; private final Monitors monitors; + private Config config; - public RecoveryRequiredChecker( FileSystemAbstraction fs, PageCache pageCache, Monitors monitors ) + public RecoveryRequiredChecker( FileSystemAbstraction fs, PageCache pageCache, Config config, Monitors monitors ) { this.fs = fs; this.pageCache = pageCache; + this.config = config; this.monitors = monitors; } @@ -62,7 +65,9 @@ public boolean isRecoveryRequiredAt( File dataDir ) throws IOException } LogEntryReader reader = new VersionAwareLogEntryReader<>(); - LogFiles logFiles = LogFilesBuilder.activeFilesBuilder( dataDir, fs, pageCache ).withLogEntryReader( reader ).build(); + LogFiles logFiles = LogFilesBuilder.activeFilesBuilder( dataDir, fs, pageCache ) + .withConfig( config ) + .withLogEntryReader( reader ).build(); LogTailScanner tailScanner = new LogTailScanner( logFiles, reader, monitors ); return new RecoveryStartInformationProvider( tailScanner, NO_MONITOR ).get().isRecoveryRequired(); } diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/storemigration/participant/StoreMigrator.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/storemigration/participant/StoreMigrator.java index dbb4516f4ccd9..32e4d64398b7b 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/storemigration/participant/StoreMigrator.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/storemigration/participant/StoreMigrator.java @@ -309,7 +309,7 @@ private TransactionId specificTransactionInformationSupplier( long lastTransacti : new TransactionId( lastTransactionId, UNKNOWN_TX_CHECKSUM, UNKNOWN_TX_COMMIT_TIMESTAMP ); } - private LogPosition extractTransactionLogPosition( File neoStore, File storeDir, long lastTxId ) throws IOException + LogPosition extractTransactionLogPosition( File neoStore, File storeDir, long lastTxId ) throws IOException { long lastClosedTxLogVersion = MetaDataStore.getRecord( pageCache, neoStore, Position.LAST_CLOSED_TRANSACTION_LOG_VERSION ); @@ -327,7 +327,9 @@ private LogPosition extractTransactionLogPosition( File neoStore, File storeDir, return new LogPosition( BASE_TX_LOG_VERSION, BASE_TX_LOG_BYTE_OFFSET ); } - LogFiles logFiles = LogFilesBuilder.activeFilesBuilder( storeDir,fileSystem, pageCache ).build(); + LogFiles logFiles = LogFilesBuilder.activeFilesBuilder( storeDir,fileSystem, pageCache ) + .withConfig( config ) + .build(); long logVersion = logFiles.getHighestLogVersion(); if ( logVersion == -1 ) { diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/transaction/log/ReadOnlyTransactionStore.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/transaction/log/ReadOnlyTransactionStore.java index e9e3353384cf6..cf51029ff02b0 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/transaction/log/ReadOnlyTransactionStore.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/transaction/log/ReadOnlyTransactionStore.java @@ -24,6 +24,7 @@ import org.neo4j.io.fs.FileSystemAbstraction; import org.neo4j.io.pagecache.PageCache; +import org.neo4j.kernel.configuration.Config; import org.neo4j.kernel.impl.transaction.log.TransactionMetadataCache.TransactionMetadata; import org.neo4j.kernel.impl.transaction.log.entry.LogEntryReader; import org.neo4j.kernel.impl.transaction.log.entry.VersionAwareLogEntryReader; @@ -41,13 +42,14 @@ public class ReadOnlyTransactionStore implements Lifecycle, LogicalTransactionSt private final LifeSupport life = new LifeSupport(); private final LogicalTransactionStore physicalStore; - public ReadOnlyTransactionStore( PageCache pageCache, FileSystemAbstraction fs, File fromPath, Monitors monitors ) - throws IOException + public ReadOnlyTransactionStore( PageCache pageCache, FileSystemAbstraction fs, File fromPath, Config config, + Monitors monitors ) throws IOException { TransactionMetadataCache transactionMetadataCache = new TransactionMetadataCache( 100 ); LogEntryReader logEntryReader = new VersionAwareLogEntryReader<>(); LogFiles logFiles = LogFilesBuilder .activeFilesBuilder( fromPath, fs, pageCache ).withLogEntryReader( logEntryReader ) + .withConfig( config ) .build(); physicalStore = new PhysicalLogicalTransactionStore( logFiles, transactionMetadataCache, logEntryReader, monitors, true ); diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/transaction/log/files/LogFiles.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/transaction/log/files/LogFiles.java index 56c8ac6175285..60eca881cb829 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/transaction/log/files/LogFiles.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/transaction/log/files/LogFiles.java @@ -39,6 +39,10 @@ public interface LogFiles extends Lifecycle File[] logFiles(); + boolean isLogFile( File file ); + + File logFilesDirectory(); + File getLogFileForVersion( long version ); File getHighestLogFile(); diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/transaction/log/files/LogFilesBuilder.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/transaction/log/files/LogFilesBuilder.java index 8880905b58707..f541f0e9f252f 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/transaction/log/files/LogFilesBuilder.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/transaction/log/files/LogFilesBuilder.java @@ -21,9 +21,11 @@ import java.io.File; import java.io.IOException; +import java.nio.file.Path; import java.util.function.LongSupplier; import java.util.function.Supplier; +import org.neo4j.graphdb.factory.GraphDatabaseSettings; import org.neo4j.io.fs.FileSystemAbstraction; import org.neo4j.io.pagecache.PageCache; import org.neo4j.kernel.configuration.Config; @@ -36,6 +38,7 @@ import org.neo4j.kernel.impl.util.Dependencies; import static java.util.Objects.requireNonNull; +import static org.neo4j.graphdb.factory.GraphDatabaseSettings.database_path; import static org.neo4j.graphdb.factory.GraphDatabaseSettings.logical_log_rotation_threshold; /** @@ -55,6 +58,7 @@ public class LogFilesBuilder private boolean readOnly; private PageCache pageCache; private File storeDirectory; + private File logsDirectory; private Config config; private Long rotationThreshold; private LogEntryReader logEntryReader; @@ -100,13 +104,15 @@ public static LogFilesBuilder activeFilesBuilder( File storeDirectory, FileSyste /** * Build log files that will be able to perform only operations on a lgo files directly. * Any operation that will require access to a store or other parts of runtime will fail. - * Should be mainly used only for testing purposes. - * @param storeDirectory store directory + * Should be mainly used only for testing purposes or when only file based operations will be performed + * @param logsDirectory log files directory * @param fileSystem file system */ - public static LogFilesBuilder logFilesBasedOnlyBuilder( File storeDirectory, FileSystemAbstraction fileSystem ) + public static LogFilesBuilder logFilesBasedOnlyBuilder( File logsDirectory, FileSystemAbstraction fileSystem ) { - LogFilesBuilder builder = builder( storeDirectory, fileSystem ); + LogFilesBuilder builder = new LogFilesBuilder(); + builder.logsDirectory = logsDirectory; + builder.fileSystem = fileSystem; builder.fileBasedOperationsOnly = true; return builder; } @@ -168,7 +174,38 @@ public LogFilesBuilder withDependencies( Dependencies dependencies ) public LogFiles build() throws IOException { TransactionLogFilesContext filesContext = buildContext(); - return new TransactionLogFiles( storeDirectory, logFileName, filesContext ); + File logsDirectory = getLogsDirectory(); + filesContext.getFileSystem().mkdirs( logsDirectory ); + return new TransactionLogFiles( logsDirectory, logFileName, filesContext ); + } + + private File getLogsDirectory() + { + if ( logsDirectory != null ) + { + return logsDirectory; + } + if ( config != null ) + { + File neo4jHome = config.get( GraphDatabaseSettings.neo4j_home ); + File databasePath = config.get( database_path ); + File logicalLogsLocation = config.get( GraphDatabaseSettings.logical_logs_location ); + if ( storeDirectory.equals( neo4jHome ) && databasePath.equals( logicalLogsLocation ) ) + { + return storeDirectory; + } + if ( logicalLogsLocation.isAbsolute() ) + { + return logicalLogsLocation; + } + if ( neo4jHome == null || !storeDirectory.equals( databasePath ) ) + { + Path relativeLogicalLogPath = databasePath.toPath().relativize( logicalLogsLocation.toPath() ); + return new File( storeDirectory, relativeLogicalLogPath.toString() ); + } + return logicalLogsLocation; + } + return storeDirectory; } TransactionLogFilesContext buildContext() throws IOException diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/transaction/log/files/TransactionLogFiles.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/transaction/log/files/TransactionLogFiles.java index 801708434aa1a..b2d05819d0f9a 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/transaction/log/files/TransactionLogFiles.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/transaction/log/files/TransactionLogFiles.java @@ -51,6 +51,7 @@ public class TransactionLogFiles extends LifecycleAdapter implements LogFiles { public static final String DEFAULT_NAME = "neostore.transaction.db"; public static final FilenameFilter DEFAULT_FILENAME_FILTER = TransactionLogFilesHelper.DEFAULT_FILENAME_FILTER; + private static final File[] EMPTY_FILES_ARRAY = {}; private final TransactionLogFilesContext logFilesContext; private final TransactionLogFileInformation logFileInformation; @@ -60,11 +61,13 @@ public class TransactionLogFiles extends LifecycleAdapter implements LogFiles private final LogFileCreationMonitor monitor; private final TransactionLogFilesHelper fileHelper; private final TransactionLogFile logFile; + private final File logsDirectory; - TransactionLogFiles( File directory, String name, TransactionLogFilesContext context ) + TransactionLogFiles( File logsDirectory, String name, TransactionLogFilesContext context ) { this.logFilesContext = context; - this.fileHelper = new TransactionLogFilesHelper( directory, name ); + this.logsDirectory = logsDirectory; + this.fileHelper = new TransactionLogFilesHelper( logsDirectory, name ); this.fileSystem = context.getFileSystem(); this.monitor = context.getLogFileCreationMonitor(); this.logHeaderCache = new LogHeaderCache( 1000 ); @@ -105,7 +108,24 @@ public long getLogVersion( String historyLogFilename ) @Override public File[] logFiles() { - return fileSystem.listFiles( fileHelper.getParentDirectory(), fileHelper.getLogFilenameFilter() ); + File[] files = fileSystem.listFiles( fileHelper.getParentDirectory(), fileHelper.getLogFilenameFilter() ); + if ( files == null ) + { + return EMPTY_FILES_ARRAY; + } + return files; + } + + @Override + public boolean isLogFile( File file ) + { + return fileHelper.getLogFilenameFilter().accept( null, file.getName() ); + } + + @Override + public File logFilesDirectory() + { + return logsDirectory; } @Override diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/transaction/log/files/TransactionLogFilesHelper.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/transaction/log/files/TransactionLogFilesHelper.java index bb37bb7c2c9c7..9627f23e87f02 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/transaction/log/files/TransactionLogFilesHelper.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/transaction/log/files/TransactionLogFilesHelper.java @@ -31,7 +31,7 @@ class TransactionLogFilesHelper private static final String VERSION_SUFFIX = "."; private static final String REGEX_VERSION_SUFFIX = "\\."; - public static final FilenameFilter DEFAULT_FILENAME_FILTER = new LogicalLogFilenameFilter( REGEX_DEFAULT_NAME ); + static final FilenameFilter DEFAULT_FILENAME_FILTER = new LogicalLogFilenameFilter( REGEX_DEFAULT_NAME ); private final File logBaseName; private final FilenameFilter logFileFilter; diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/transaction/state/NeoStoreFileListing.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/transaction/state/NeoStoreFileListing.java index d27476f2bde9f..79246a28438a4 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/transaction/state/NeoStoreFileListing.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/transaction/state/NeoStoreFileListing.java @@ -34,9 +34,9 @@ import org.neo4j.kernel.impl.api.ExplicitIndexProviderLookup; import org.neo4j.kernel.impl.api.index.IndexingService; import org.neo4j.kernel.impl.index.IndexConfigStore; -import org.neo4j.kernel.impl.store.MetaDataStore; import org.neo4j.kernel.impl.store.StoreType; import org.neo4j.kernel.impl.store.format.RecordFormat; +import org.neo4j.kernel.impl.transaction.log.files.LogFiles; import org.neo4j.kernel.spi.explicitindex.IndexImplementation; import org.neo4j.storageengine.api.StorageEngine; import org.neo4j.storageengine.api.StoreFileMetadata; @@ -47,17 +47,22 @@ public class NeoStoreFileListing { private final File storeDir; + private final LogFiles logFiles; private final LabelScanStore labelScanStore; private final IndexingService indexingService; private final ExplicitIndexProviderLookup explicitIndexProviders; private final StorageEngine storageEngine; private final Function toNotAStoreTypeFile = file -> new StoreFileMetadata( file, RecordFormat.NO_RECORD_SIZE ); + private final Function logFileMapper = + file -> new StoreFileMetadata( file, RecordFormat.NO_RECORD_SIZE, true ); - public NeoStoreFileListing( File storeDir, LabelScanStore labelScanStore, IndexingService indexingService, + public NeoStoreFileListing( File storeDir, LogFiles logFiles, + LabelScanStore labelScanStore, IndexingService indexingService, ExplicitIndexProviderLookup explicitIndexProviders, StorageEngine storageEngine ) { this.storeDir = storeDir; + this.logFiles = logFiles; this.labelScanStore = labelScanStore; this.indexingService = indexingService; this.explicitIndexProviders = explicitIndexProviders; @@ -100,20 +105,20 @@ private void placeMetaDataStoreLast( List files ) private void gatherNonRecordStores( Collection files, boolean includeLogs ) { - final File[] storeFiles = storeDir.listFiles(); - if ( storeFiles == null ) + File[] indexFiles = storeDir.listFiles( ( dir, name ) -> name.equals( IndexConfigStore.INDEX_DB_FILE_NAME ) ); + if ( indexFiles != null ) { - return; - } - for ( File file : storeFiles ) - { - if ( file.getName().equals( IndexConfigStore.INDEX_DB_FILE_NAME ) ) + for ( File file : indexFiles ) { files.add( toNotAStoreTypeFile.apply( file ) ); } - else if ( includeLogs && transactionLogFile( file.getName() ) ) + } + if ( includeLogs ) + { + File[] logFiles = this.logFiles.logFiles(); + for ( File logFile : logFiles ) { - files.add( toNotAStoreTypeFile.apply( file ) ); + files.add( logFileMapper.apply( logFile ) ); } } } @@ -155,11 +160,6 @@ private void gatherNeoStoreFiles( final Collection targetFile targetFiles.addAll( storageEngine.listStorageFiles() ); } - private boolean transactionLogFile( String name ) - { - return name.startsWith( MetaDataStore.DEFAULT_NAME + ".transaction" ) && !name.endsWith( ".active" ); - } - private static final class MultiResource implements Resource { private final Collection snapshots; diff --git a/community/kernel/src/main/java/org/neo4j/storageengine/api/StoreFileMetadata.java b/community/kernel/src/main/java/org/neo4j/storageengine/api/StoreFileMetadata.java index 78392c5f84890..1dd5a07b05616 100644 --- a/community/kernel/src/main/java/org/neo4j/storageengine/api/StoreFileMetadata.java +++ b/community/kernel/src/main/java/org/neo4j/storageengine/api/StoreFileMetadata.java @@ -25,11 +25,18 @@ public class StoreFileMetadata { private final File file; private final int recordSize; + private final boolean isLogFile; public StoreFileMetadata( File file, int recordSize ) + { + this( file, recordSize, false ); + } + + public StoreFileMetadata( File file, int recordSize, boolean isLogFile ) { this.file = file; this.recordSize = recordSize; + this.isLogFile = isLogFile; } public File file() @@ -41,4 +48,9 @@ public int recordSize() { return recordSize; } + + public boolean isLogFile() + { + return isLogFile; + } } diff --git a/community/kernel/src/test/java/org/neo4j/graphdb/TransactionLogsInSeparateLocationIT.java b/community/kernel/src/test/java/org/neo4j/graphdb/TransactionLogsInSeparateLocationIT.java new file mode 100644 index 0000000000000..93c70428afcfc --- /dev/null +++ b/community/kernel/src/test/java/org/neo4j/graphdb/TransactionLogsInSeparateLocationIT.java @@ -0,0 +1,100 @@ +/* + * Copyright (c) 2002-2017 "Neo Technology," + * Network Engine for Objects in Lund AB [http://neotechnology.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.graphdb; + +import org.junit.Rule; +import org.junit.Test; + +import java.io.File; +import java.io.IOException; + +import org.neo4j.graphdb.factory.GraphDatabaseSettings; +import org.neo4j.io.fs.FileSystemAbstraction; +import org.neo4j.kernel.impl.transaction.log.PhysicalLogVersionedStoreChannel; +import org.neo4j.kernel.impl.transaction.log.files.LogFiles; +import org.neo4j.kernel.impl.transaction.log.files.LogFilesBuilder; +import org.neo4j.test.TestGraphDatabaseFactory; +import org.neo4j.test.rule.TestDirectory; +import org.neo4j.test.rule.fs.DefaultFileSystemRule; +import org.neo4j.test.rule.fs.FileSystemRule; + +import static org.hamcrest.Matchers.greaterThan; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertTrue; + +public class TransactionLogsInSeparateLocationIT +{ + @Rule + public final TestDirectory testDirectory = TestDirectory.testDirectory(); + @Rule + public final FileSystemRule fileSystemRule = new DefaultFileSystemRule(); + + @Test + public void databaseWithTransactionLogsInSeparateRelativeLocation() throws IOException + { + File storeDir = testDirectory.graphDbDir(); + File txDirectory = new File( storeDir, "transaction-logs" ); + performTransactions( txDirectory.getName(), storeDir ); + verifyTransactionLogs( txDirectory, storeDir ); + } + + @Test + public void databaseWithTransactionLogsInSeparateAbsoluteLocation() throws IOException + { + File storeDir = testDirectory.graphDbDir(); + File txDirectory = testDirectory.directory( "transaction-logs" ); + performTransactions( txDirectory.getAbsolutePath(), storeDir ); + verifyTransactionLogs( txDirectory, storeDir ); + } + + private void performTransactions( String txPath, File storeDir ) throws IOException + { + GraphDatabaseService database = new TestGraphDatabaseFactory().newEmbeddedDatabaseBuilder( storeDir ) + .setConfig( GraphDatabaseSettings.logical_logs_location, txPath ) + .newGraphDatabase(); + for ( int i = 0; i < 10; i++ ) + { + try ( Transaction transaction = database.beginTx() ) + { + Node node = database.createNode(); + node.setProperty( "a", "b" ); + node.setProperty( "c", "d" ); + transaction.success(); + } + } + database.shutdown(); + } + + private void verifyTransactionLogs( File txDirectory, File storeDir ) throws IOException + { + FileSystemAbstraction fileSystem = fileSystemRule.get(); + LogFiles storeDirLogs = LogFilesBuilder.logFilesBasedOnlyBuilder( storeDir, fileSystem ).build(); + assertFalse( storeDirLogs.versionExists( 0 ) ); + + LogFiles txDirectoryLogs = LogFilesBuilder.logFilesBasedOnlyBuilder( txDirectory, fileSystem ).build(); + assertTrue( txDirectoryLogs.versionExists( 0 ) ); + try ( PhysicalLogVersionedStoreChannel physicalLogVersionedStoreChannel = txDirectoryLogs.openForVersion( 0 ) ) + { + assertThat( physicalLogVersionedStoreChannel.size(), greaterThan( 0L ) ); + } + } + +} diff --git a/community/kernel/src/test/java/org/neo4j/kernel/impl/recovery/RecoveryRequiredCheckerTest.java b/community/kernel/src/test/java/org/neo4j/kernel/impl/recovery/RecoveryRequiredCheckerTest.java index b1579267c7ace..7faf0d9cbab98 100644 --- a/community/kernel/src/test/java/org/neo4j/kernel/impl/recovery/RecoveryRequiredCheckerTest.java +++ b/community/kernel/src/test/java/org/neo4j/kernel/impl/recovery/RecoveryRequiredCheckerTest.java @@ -32,6 +32,7 @@ import org.neo4j.graphdb.mockfs.EphemeralFileSystemAbstraction; import org.neo4j.io.fs.FileSystemAbstraction; import org.neo4j.io.pagecache.PageCache; +import org.neo4j.kernel.configuration.Config; import org.neo4j.kernel.monitoring.Monitors; import org.neo4j.test.TestGraphDatabaseFactory; import org.neo4j.test.rule.PageCacheRule; @@ -40,6 +41,7 @@ import static org.hamcrest.CoreMatchers.is; import static org.junit.Assert.assertThat; +import static org.neo4j.graphdb.factory.GraphDatabaseSettings.logical_logs_location; public class RecoveryRequiredCheckerTest { @@ -66,7 +68,7 @@ public void setup() public void shouldNotWantToRecoverIntactStore() throws Exception { PageCache pageCache = pageCacheRule.getPageCache( fileSystem ); - RecoveryRequiredChecker recoverer = getRecoveryChecker( fileSystem, pageCache ); + RecoveryRequiredChecker recoverer = getRecoveryCheckerWithDefaultConfig( fileSystem, pageCache ); assertThat( recoverer.isRecoveryRequiredAt( storeDir ), is( false ) ); } @@ -74,11 +76,11 @@ public void shouldNotWantToRecoverIntactStore() throws Exception @Test public void shouldWantToRecoverBrokenStore() throws Exception { - try ( FileSystemAbstraction fileSystemAbstraction = createSomeDataAndCrash( storeDir, fileSystem ) ) + try ( FileSystemAbstraction fileSystemAbstraction = createAndCrashWithDefaultConfig() ) { PageCache pageCache = pageCacheRule.getPageCache( fileSystemAbstraction ); - RecoveryRequiredChecker recoverer = getRecoveryChecker( fileSystemAbstraction, pageCache ); + RecoveryRequiredChecker recoverer = getRecoveryCheckerWithDefaultConfig( fileSystemAbstraction, pageCache ); assertThat( recoverer.isRecoveryRequiredAt( storeDir ), is( true ) ); } @@ -87,11 +89,11 @@ public void shouldWantToRecoverBrokenStore() throws Exception @Test public void shouldBeAbleToRecoverBrokenStore() throws Exception { - try ( FileSystemAbstraction fileSystemAbstraction = createSomeDataAndCrash( storeDir, fileSystem ) ) + try ( FileSystemAbstraction fileSystemAbstraction = createAndCrashWithDefaultConfig() ) { PageCache pageCache = pageCacheRule.getPageCache( fileSystemAbstraction ); - RecoveryRequiredChecker recoverer = getRecoveryChecker( fileSystemAbstraction, pageCache ); + RecoveryRequiredChecker recoverer = getRecoveryCheckerWithDefaultConfig( fileSystemAbstraction, pageCache ); assertThat( recoverer.isRecoveryRequiredAt( storeDir ), is( true ) ); @@ -101,16 +103,66 @@ public void shouldBeAbleToRecoverBrokenStore() throws Exception } } - private RecoveryRequiredChecker getRecoveryChecker( FileSystemAbstraction fileSystem, PageCache pageCache ) + @Test + public void shouldBeAbleToRecoverBrokenStoreWithLogsInSeparateRelativeLocation() throws Exception + { + File customTransactionLogsLocation = new File( storeDir, "tx-logs" ); + Config config = Config.defaults( logical_logs_location, customTransactionLogsLocation.getName() ); + recoverBrokenStoreWithConfig( config ); + } + + @Test + public void shouldBeAbleToRecoverBrokenStoreWithLogsInSeparateAbsoluteLocation() throws Exception + { + File customTransactionLogsLocation = testDirectory.directory( "tx-logs" ); + Config config = Config.defaults( logical_logs_location, customTransactionLogsLocation.getAbsolutePath() ); + recoverBrokenStoreWithConfig( config ); + } + + private void recoverBrokenStoreWithConfig( Config config ) throws IOException + { + try ( FileSystemAbstraction fileSystemAbstraction = createSomeDataAndCrash( storeDir, fileSystem, config ) ) + { + PageCache pageCache = pageCacheRule.getPageCache( fileSystemAbstraction ); + + RecoveryRequiredChecker recoverer = getRecoveryChecker( fileSystemAbstraction, pageCache, config ); + + assertThat( recoverer.isRecoveryRequiredAt( storeDir ), is( true ) ); + + new TestGraphDatabaseFactory() + .setFileSystem( fileSystemAbstraction ) + .newEmbeddedDatabaseBuilder( storeDir ) + .setConfig( config.getRaw() ) + .newGraphDatabase() + .shutdown(); + + assertThat( recoverer.isRecoveryRequiredAt( storeDir ), is( false ) ); + } + } + + private FileSystemAbstraction createAndCrashWithDefaultConfig() throws IOException + { + return createSomeDataAndCrash( storeDir, fileSystem, Config.defaults() ); + } + + private RecoveryRequiredChecker getRecoveryCheckerWithDefaultConfig( FileSystemAbstraction fileSystem, PageCache pageCache ) + { + return getRecoveryChecker( fileSystem, pageCache, Config.defaults() ); + } + + private RecoveryRequiredChecker getRecoveryChecker( FileSystemAbstraction fileSystem, PageCache pageCache, Config config ) { - return new RecoveryRequiredChecker( fileSystem, pageCache, monitors ); + return new RecoveryRequiredChecker( fileSystem, pageCache, config, monitors ); } - private FileSystemAbstraction createSomeDataAndCrash( File store, EphemeralFileSystemAbstraction fileSystem ) - throws IOException + private FileSystemAbstraction createSomeDataAndCrash( File store, EphemeralFileSystemAbstraction fileSystem, + Config config ) { - final GraphDatabaseService db = - new TestGraphDatabaseFactory().setFileSystem( fileSystem ).newImpermanentDatabase( store ); + final GraphDatabaseService db = new TestGraphDatabaseFactory() + .setFileSystem( fileSystem ) + .newImpermanentDatabaseBuilder( store ) + .setConfig( config.getRaw() ) + .newGraphDatabase(); try ( Transaction tx = db.beginTx() ) { diff --git a/community/kernel/src/test/java/org/neo4j/kernel/impl/store/TestStoreAccess.java b/community/kernel/src/test/java/org/neo4j/kernel/impl/store/TestStoreAccess.java index a17851f19ede7..75560f63a899d 100644 --- a/community/kernel/src/test/java/org/neo4j/kernel/impl/store/TestStoreAccess.java +++ b/community/kernel/src/test/java/org/neo4j/kernel/impl/store/TestStoreAccess.java @@ -81,7 +81,7 @@ private EphemeralFileSystemAbstraction produceUncleanStore() private boolean isUnclean( FileSystemAbstraction fileSystem ) throws IOException { PageCache pageCache = pageCacheRule.getPageCache( fileSystem ); - - return new RecoveryRequiredChecker( fileSystem, pageCache, monitors ).isRecoveryRequiredAt( storeDir ); + RecoveryRequiredChecker requiredChecker = new RecoveryRequiredChecker( fileSystem, pageCache, Config.defaults(), monitors ); + return requiredChecker.isRecoveryRequiredAt( storeDir ); } } diff --git a/community/kernel/src/test/java/org/neo4j/kernel/impl/storemigration/participant/StoreMigratorTest.java b/community/kernel/src/test/java/org/neo4j/kernel/impl/storemigration/participant/StoreMigratorTest.java index 11fcbe4f185eb..9e9aac4bbc9a1 100644 --- a/community/kernel/src/test/java/org/neo4j/kernel/impl/storemigration/participant/StoreMigratorTest.java +++ b/community/kernel/src/test/java/org/neo4j/kernel/impl/storemigration/participant/StoreMigratorTest.java @@ -27,18 +27,24 @@ import java.io.File; import java.io.IOException; +import org.neo4j.graphdb.GraphDatabaseService; +import org.neo4j.graphdb.Node; +import org.neo4j.graphdb.Transaction; import org.neo4j.io.pagecache.PageCache; import org.neo4j.kernel.configuration.Config; import org.neo4j.kernel.impl.logging.LogService; import org.neo4j.kernel.impl.logging.NullLogService; import org.neo4j.kernel.impl.logging.SimpleLogService; +import org.neo4j.kernel.impl.store.MetaDataStore; import org.neo4j.kernel.impl.store.TransactionId; +import org.neo4j.kernel.impl.store.format.standard.MetaDataRecordFormat; import org.neo4j.kernel.impl.store.format.standard.StandardV3_0; import org.neo4j.kernel.impl.store.format.standard.StandardV3_2; import org.neo4j.kernel.impl.transaction.log.LogPosition; import org.neo4j.kernel.impl.transaction.log.TransactionIdStore; import org.neo4j.kernel.impl.util.monitoring.ProgressReporter; import org.neo4j.logging.NullLogProvider; +import org.neo4j.test.TestGraphDatabaseFactory; import org.neo4j.test.rule.PageCacheRule; import org.neo4j.test.rule.RandomRule; import org.neo4j.test.rule.TestDirectory; @@ -46,7 +52,9 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; import static org.mockito.Mockito.mock; +import static org.neo4j.graphdb.factory.GraphDatabaseSettings.logical_logs_location; import static org.neo4j.kernel.impl.store.MetaDataStore.DEFAULT_NAME; import static org.neo4j.kernel.impl.store.MetaDataStore.Position.LAST_TRANSACTION_CHECKSUM; import static org.neo4j.kernel.impl.store.MetaDataStore.Position.LAST_TRANSACTION_COMMIT_TIMESTAMP; @@ -134,6 +142,51 @@ public void shouldGenerateTransactionInformationWhenLogsNotPresent() throws Exce assertEquals( TransactionIdStore.UNKNOWN_TX_COMMIT_TIMESTAMP, actual.commitTimestamp() ); } + @Test + public void extractTransactionInformationFromLogsInCustomRelativeLocation() throws Exception + { + File storeDir = directory.graphDbDir(); + File customLogLocation = new File( storeDir, "customLogLocation" ); + extractTransactionalInformationFromLogs( customLogLocation.getName(), customLogLocation, storeDir ); + } + + @Test + public void extractTransactionInformationFromLogsInCustomAbsoluteLocation() throws Exception + { + File storeDir = directory.graphDbDir(); + File customLogLocation = directory.directory( "customLogLocation" ); + extractTransactionalInformationFromLogs( customLogLocation.getAbsolutePath(), customLogLocation, storeDir ); + } + + private void extractTransactionalInformationFromLogs( String path, File customLogLocation, File storeDir ) throws IOException + { + LogService logService = new SimpleLogService( NullLogProvider.getInstance(), NullLogProvider.getInstance() ); + File neoStore = new File( storeDir, DEFAULT_NAME ); + + GraphDatabaseService database = new TestGraphDatabaseFactory().newEmbeddedDatabaseBuilder( storeDir ) + .setConfig( logical_logs_location, path ).newGraphDatabase(); + for ( int i = 0; i < 10; i++ ) + { + try ( Transaction transaction = database.beginTx() ) + { + Node node = database.createNode(); + transaction.success(); + } + } + database.shutdown(); + + MetaDataStore.setRecord( pageCache, neoStore, MetaDataStore.Position.LAST_CLOSED_TRANSACTION_LOG_VERSION, + MetaDataRecordFormat.FIELD_NOT_PRESENT ); + Config config = Config.defaults( logical_logs_location, path ); + StoreMigrator migrator = new StoreMigrator( fileSystemRule.get(), pageCache, config, logService ); + LogPosition logPosition = migrator.extractTransactionLogPosition( neoStore, storeDir, 100 ); + + File[] logFiles = customLogLocation.listFiles(); + assertNotNull( logFiles ); + assertEquals( 0, logPosition.getLogVersion() ); + assertEquals( logFiles[0].length(), logPosition.getByteOffset() ); + } + @Test public void shouldGenerateTransactionInformationWhenLogsAreEmpty() throws Exception { diff --git a/community/kernel/src/test/java/org/neo4j/kernel/impl/transaction/log/files/LogFilesBuilderTest.java b/community/kernel/src/test/java/org/neo4j/kernel/impl/transaction/log/files/LogFilesBuilderTest.java index bc8463b6e0cef..35f8c6f3fcfb4 100644 --- a/community/kernel/src/test/java/org/neo4j/kernel/impl/transaction/log/files/LogFilesBuilderTest.java +++ b/community/kernel/src/test/java/org/neo4j/kernel/impl/transaction/log/files/LogFilesBuilderTest.java @@ -26,19 +26,21 @@ import java.io.File; import java.io.IOException; -import org.neo4j.graphdb.mockfs.EphemeralFileSystemAbstraction; import org.neo4j.io.ByteUnit; +import org.neo4j.io.fs.DefaultFileSystemAbstraction; import org.neo4j.io.pagecache.PageCache; +import org.neo4j.kernel.configuration.Config; import org.neo4j.kernel.impl.transaction.SimpleLogVersionRepository; import org.neo4j.kernel.impl.transaction.SimpleTransactionIdStore; import org.neo4j.kernel.impl.util.Dependencies; import org.neo4j.test.rule.PageCacheRule; import org.neo4j.test.rule.TestDirectory; -import org.neo4j.test.rule.fs.EphemeralFileSystemRule; +import org.neo4j.test.rule.fs.DefaultFileSystemRule; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertSame; +import static org.neo4j.graphdb.factory.GraphDatabaseSettings.logical_logs_location; import static org.neo4j.kernel.impl.transaction.log.files.LogFilesBuilder.activeFilesBuilder; import static org.neo4j.kernel.impl.transaction.log.files.LogFilesBuilder.builder; import static org.neo4j.kernel.impl.transaction.log.files.LogFilesBuilder.logFilesBasedOnlyBuilder; @@ -48,17 +50,17 @@ public class LogFilesBuilderTest @Rule public final TestDirectory testDirectory = TestDirectory.testDirectory(); @Rule - public final EphemeralFileSystemRule fileSystemRule = new EphemeralFileSystemRule(); + public final DefaultFileSystemRule fileSystemRule = new DefaultFileSystemRule(); @Rule public final PageCacheRule pageCacheRule = new PageCacheRule(); - private File directory; - private EphemeralFileSystemAbstraction fileSystem; + private File storeDirectory; + private DefaultFileSystemAbstraction fileSystem; @Before public void setUp() throws Exception { - directory = testDirectory.directory(); + storeDirectory = testDirectory.directory(); fileSystem = fileSystemRule.get(); } @@ -66,7 +68,7 @@ public void setUp() throws Exception public void buildActiveFilesOnlyContext() throws IOException { PageCache pageCache = pageCacheRule.getPageCache( fileSystem ); - TransactionLogFilesContext context = activeFilesBuilder( directory, fileSystem, pageCache ).buildContext(); + TransactionLogFilesContext context = activeFilesBuilder( storeDirectory, fileSystem, pageCache ).buildContext(); assertEquals( fileSystem, context.getFileSystem() ); assertNotNull( context.getLogEntryReader() ); @@ -79,7 +81,7 @@ public void buildActiveFilesOnlyContext() throws IOException @Test public void buildFilesBasedContext() throws IOException { - TransactionLogFilesContext context = logFilesBasedOnlyBuilder( directory, fileSystem ).buildContext(); + TransactionLogFilesContext context = logFilesBasedOnlyBuilder( storeDirectory, fileSystem ).buildContext(); assertEquals( fileSystem, context.getFileSystem() ); assertSame( LogFileCreationMonitor.NO_MONITOR, context.getLogFileCreationMonitor() ); } @@ -87,8 +89,8 @@ public void buildFilesBasedContext() throws IOException @Test public void buildDefaultContext() throws IOException { - TransactionLogFilesContext context = - builder( directory, fileSystem ).withLogVersionRepository( new SimpleLogVersionRepository( 2 ) ) + TransactionLogFilesContext context = builder( storeDirectory, fileSystem ) + .withLogVersionRepository( new SimpleLogVersionRepository( 2 ) ) .withTransactionIdStore( new SimpleTransactionIdStore() ).buildContext(); assertEquals( fileSystem, context.getFileSystem() ); assertNotNull( context.getLogEntryReader() ); @@ -108,7 +110,7 @@ public void buildDefaultContextWithDependencies() throws IOException dependencies.satisfyDependency( transactionIdStore ); TransactionLogFilesContext context = - builder( directory, fileSystem ).withDependencies( dependencies ).buildContext(); + builder( storeDirectory, fileSystem ).withDependencies( dependencies ).buildContext(); assertEquals( fileSystem, context.getFileSystem() ); assertNotNull( context.getLogEntryReader() ); @@ -118,27 +120,55 @@ public void buildDefaultContextWithDependencies() throws IOException assertEquals( 2, context.getLogVersionRepository().getCurrentLogVersion() ); } + @Test + public void buildContextWithCustomLogFilesLocations() throws Throwable + { + String customLogLocation = "customLogLocation"; + Config customLogLocationConfig = Config.defaults( logical_logs_location, customLogLocation ); + LogFiles logFiles = builder( storeDirectory, fileSystem ).withConfig( customLogLocationConfig ) + .withLogVersionRepository( new SimpleLogVersionRepository() ) + .withTransactionIdStore( new SimpleTransactionIdStore() ).build(); + logFiles.init(); + logFiles.start(); + + assertEquals( new File( storeDirectory, customLogLocation ), logFiles.getHighestLogFile().getParentFile() ); + } + + @Test + public void buildContextWithCustomAbsoluteLogFilesLocations() throws Throwable + { + File customLogDirectory = testDirectory.directory( "absoluteCustomLogDirectory" ); + Config customLogLocationConfig = Config.defaults( logical_logs_location, customLogDirectory.getAbsolutePath() ); + LogFiles logFiles = builder( storeDirectory, fileSystem ).withConfig( customLogLocationConfig ) + .withLogVersionRepository( new SimpleLogVersionRepository() ) + .withTransactionIdStore( new SimpleTransactionIdStore() ).build(); + logFiles.init(); + logFiles.start(); + + assertEquals( customLogDirectory, logFiles.getHighestLogFile().getParentFile() ); + } + @Test( expected = NullPointerException.class ) public void failToBuildFullContextWithoutLogVersionRepo() throws IOException { - builder( directory, fileSystem ).withTransactionIdStore( new SimpleTransactionIdStore() ).buildContext(); + builder( storeDirectory, fileSystem ).withTransactionIdStore( new SimpleTransactionIdStore() ).buildContext(); } @Test( expected = NullPointerException.class ) public void failToBuildFullContextWithoutTransactionIdStore() throws IOException { - builder( directory, fileSystem ).withLogVersionRepository( new SimpleLogVersionRepository( 2 ) ).buildContext(); + builder( storeDirectory, fileSystem ).withLogVersionRepository( new SimpleLogVersionRepository( 2 ) ).buildContext(); } @Test( expected = UnsupportedOperationException.class ) public void fileBasedOperationsContextFailOnLastCommittedTransactionIdAccess() throws IOException { - logFilesBasedOnlyBuilder( directory, fileSystem ).buildContext().getLastCommittedTransactionId(); + logFilesBasedOnlyBuilder( storeDirectory, fileSystem ).buildContext().getLastCommittedTransactionId(); } @Test( expected = UnsupportedOperationException.class ) public void fileBasedOperationsContextFailOnLogVersionRepositoryAccess() throws IOException { - logFilesBasedOnlyBuilder( directory, fileSystem ).buildContext().getLogVersionRepository(); + logFilesBasedOnlyBuilder( storeDirectory, fileSystem ).buildContext().getLogVersionRepository(); } } diff --git a/community/kernel/src/test/java/org/neo4j/kernel/impl/transaction/log/files/TransactionLogFilesTest.java b/community/kernel/src/test/java/org/neo4j/kernel/impl/transaction/log/files/TransactionLogFilesTest.java index bbc1c5d426ba9..de419f073ace3 100644 --- a/community/kernel/src/test/java/org/neo4j/kernel/impl/transaction/log/files/TransactionLogFilesTest.java +++ b/community/kernel/src/test/java/org/neo4j/kernel/impl/transaction/log/files/TransactionLogFilesTest.java @@ -36,7 +36,9 @@ import static org.hamcrest.Matchers.containsInAnyOrder; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; public class TransactionLogFilesTest @@ -45,13 +47,13 @@ public class TransactionLogFilesTest public final TestDirectory testDirectory = TestDirectory.testDirectory(); @Rule public final FileSystemRule fileSystemRule = new DefaultFileSystemRule(); - private File logDirectory; + private File storeDirectory; private final String filename = "filename"; @Before public void setUp() throws Exception { - logDirectory = testDirectory.directory(); + storeDirectory = testDirectory.directory(); } @Test @@ -65,7 +67,7 @@ public void shouldGetTheFileNameForAGivenVersion() throws IOException final File versionFileName = files.getLogFileForVersion( version ); // then - final File expected = new File( logDirectory, getVersionedLogFileName( version ) ); + final File expected = new File( storeDirectory, getVersionedLogFileName( version ) ); assertEquals( expected, versionFileName ); } @@ -75,10 +77,10 @@ public void shouldVisitEachLofFile() throws IOException // given LogFiles files = createLogFiles(); - fileSystemRule.create( new File( logDirectory, getVersionedLogFileName( "1" ) ) ); - fileSystemRule.create( new File( logDirectory, getVersionedLogFileName( "some", "2" ) ) ); - fileSystemRule.create( new File( logDirectory, getVersionedLogFileName( "3" ) ) ); - fileSystemRule.create( new File( logDirectory, filename ) ); + fileSystemRule.create( new File( storeDirectory, getVersionedLogFileName( "1" ) ) ); + fileSystemRule.create( new File( storeDirectory, getVersionedLogFileName( "some", "2" ) ) ); + fileSystemRule.create( new File( storeDirectory, getVersionedLogFileName( "3" ) ) ); + fileSystemRule.create( new File( storeDirectory, filename ) ); // when final List seenFiles = new ArrayList<>(); @@ -92,8 +94,8 @@ public void shouldVisitEachLofFile() throws IOException // then assertThat( seenFiles, containsInAnyOrder( - new File( logDirectory, getVersionedLogFileName( filename, "1" ) ), - new File( logDirectory, getVersionedLogFileName( filename, "3" ) ) ) ); + new File( storeDirectory, getVersionedLogFileName( filename, "1" ) ), + new File( storeDirectory, getVersionedLogFileName( filename, "3" ) ) ) ); assertThat( seenVersions, containsInAnyOrder( 1L, 3L ) ); } @@ -103,10 +105,10 @@ public void shouldBeAbleToRetrieveTheHighestLogVersion() throws IOException // given LogFiles files = createLogFiles(); - fileSystemRule.create( new File( logDirectory, getVersionedLogFileName( "1" ) ) ); - fileSystemRule.create( new File( logDirectory, getVersionedLogFileName( "some", "4" ) ) ); - fileSystemRule.create( new File( logDirectory, getVersionedLogFileName( "3" ) ) ); - fileSystemRule.create( new File( logDirectory, filename ) ); + fileSystemRule.create( new File( storeDirectory, getVersionedLogFileName( "1" ) ) ); + fileSystemRule.create( new File( storeDirectory, getVersionedLogFileName( "some", "4" ) ) ); + fileSystemRule.create( new File( storeDirectory, getVersionedLogFileName( "3" ) ) ); + fileSystemRule.create( new File( storeDirectory, filename ) ); // when final long highestLogVersion = files.getHighestLogVersion(); @@ -121,8 +123,8 @@ public void shouldReturnANegativeValueIfThereAreNoLogFiles() throws IOException // given LogFiles files = createLogFiles(); - fileSystemRule.create( new File( logDirectory, getVersionedLogFileName( "some", "4" ) ) ); - fileSystemRule.create( new File( logDirectory, filename ) ); + fileSystemRule.create( new File( storeDirectory, getVersionedLogFileName( "some", "4" ) ) ); + fileSystemRule.create( new File( storeDirectory, filename ) ); // when final long highestLogVersion = files.getHighestLogVersion(); @@ -174,10 +176,19 @@ public void shouldThrowIfVersionIsNotANumber() throws IOException logFiles.getLogVersion( file ); } + @Test + public void isLogFile() throws IOException + { + LogFiles logFiles = createLogFiles(); + assertFalse( logFiles.isLogFile( new File( "aaa.tx.log" ) ) ); + assertTrue( logFiles.isLogFile( new File( "filename.0" ) ) ); + assertTrue( logFiles.isLogFile( new File( "filename.17" ) ) ); + } + private LogFiles createLogFiles() throws IOException { return LogFilesBuilder - .builder( logDirectory, fileSystemRule ) + .builder( storeDirectory, fileSystemRule ) .withLogFileName( filename ) .withTransactionIdStore( new SimpleTransactionIdStore() ) .withLogVersionRepository( new SimpleLogVersionRepository() ) diff --git a/community/kernel/src/test/java/org/neo4j/kernel/impl/transaction/log/stresstest/workload/Runner.java b/community/kernel/src/test/java/org/neo4j/kernel/impl/transaction/log/stresstest/workload/Runner.java index 3f3c8f5fbb041..2205581c435b8 100644 --- a/community/kernel/src/test/java/org/neo4j/kernel/impl/transaction/log/stresstest/workload/Runner.java +++ b/community/kernel/src/test/java/org/neo4j/kernel/impl/transaction/log/stresstest/workload/Runner.java @@ -70,7 +70,7 @@ public Long call() throws Exception { TransactionIdStore transactionIdStore = new SimpleTransactionIdStore(); TransactionMetadataCache transactionMetadataCache = new TransactionMetadataCache( 100_000 ); - LogFiles logFiles = life.add( createPhysicalLogFiles( transactionIdStore, fileSystem ) ); + LogFiles logFiles = life.add( createLogFiles( transactionIdStore, fileSystem ) ); TransactionAppender transactionAppender = life.add( createBatchingTransactionAppender( transactionIdStore, transactionMetadataCache, logFiles ) ); @@ -115,7 +115,7 @@ private BatchingTransactionAppender createBatchingTransactionAppender( Transacti transactionMetadataCache, transactionIdStore, IdOrderingQueue.BYPASS, databaseHealth ); } - private LogFiles createPhysicalLogFiles( TransactionIdStore transactionIdStore, + private LogFiles createLogFiles( TransactionIdStore transactionIdStore, FileSystemAbstraction fileSystemAbstraction ) throws IOException { SimpleLogVersionRepository logVersionRepository = new SimpleLogVersionRepository(); diff --git a/community/kernel/src/test/java/org/neo4j/kernel/impl/transaction/state/NeoStoreFileListingTest.java b/community/kernel/src/test/java/org/neo4j/kernel/impl/transaction/state/NeoStoreFileListingTest.java index 055526c4b9cee..baf9b45471cc6 100644 --- a/community/kernel/src/test/java/org/neo4j/kernel/impl/transaction/state/NeoStoreFileListingTest.java +++ b/community/kernel/src/test/java/org/neo4j/kernel/impl/transaction/state/NeoStoreFileListingTest.java @@ -25,6 +25,7 @@ import java.io.File; import java.io.IOException; +import java.nio.file.Paths; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -34,15 +35,20 @@ import java.util.stream.Collectors; import org.neo4j.graphdb.ResourceIterator; +import org.neo4j.graphdb.factory.GraphDatabaseSettings; import org.neo4j.kernel.NeoStoreDataSource; import org.neo4j.kernel.api.labelscan.LabelScanStore; import org.neo4j.kernel.impl.api.ExplicitIndexProviderLookup; import org.neo4j.kernel.impl.api.index.IndexingService; import org.neo4j.kernel.impl.store.StoreType; +import org.neo4j.kernel.impl.transaction.log.files.LogFiles; import org.neo4j.kernel.impl.transaction.log.files.TransactionLogFiles; +import org.neo4j.kernel.internal.GraphDatabaseAPI; import org.neo4j.storageengine.api.StorageEngine; import org.neo4j.storageengine.api.StoreFileMetadata; +import org.neo4j.test.TestGraphDatabaseFactory; import org.neo4j.test.rule.EmbeddedDatabaseRule; +import org.neo4j.test.rule.TestDirectory; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; @@ -55,7 +61,10 @@ public class NeoStoreFileListingTest { @Rule - public EmbeddedDatabaseRule db = new EmbeddedDatabaseRule(); + public final EmbeddedDatabaseRule db = new EmbeddedDatabaseRule(); + @Rule + public final TestDirectory testDirectory = TestDirectory.testDirectory(); + private NeoStoreDataSource neoStoreDataSource; private static final String[] STANDARD_STORE_DIR_FILES = new String[]{ "index", @@ -110,16 +119,6 @@ public void setUp() throws IOException neoStoreDataSource = db.getDependencyResolver().resolveDependency( NeoStoreDataSource.class ); } - private void createIndexDbFile() throws IOException - { - File storeDir = db.getStoreDir(); - final File indexFile = new File( storeDir, "index.db" ); - if ( !indexFile.exists() ) - { - assertTrue( indexFile.createNewFile() ); - } - } - @Test public void shouldCloseIndexAndLabelScanSnapshots() throws Exception { @@ -129,10 +128,11 @@ public void shouldCloseIndexAndLabelScanSnapshots() throws Exception ExplicitIndexProviderLookup explicitIndexes = mock( ExplicitIndexProviderLookup.class ); when( explicitIndexes.all() ).thenReturn( Collections.emptyList() ); File storeDir = mock( File.class ); + LogFiles logFiles = mock( LogFiles.class ); filesInStoreDirAre( storeDir, STANDARD_STORE_DIR_FILES, STANDARD_STORE_DIR_DIRECTORIES ); StorageEngine storageEngine = mock( StorageEngine.class ); - NeoStoreFileListing fileListing = new NeoStoreFileListing( - storeDir, labelScanStore, indexingService, explicitIndexes, storageEngine ); + NeoStoreFileListing fileListing = new NeoStoreFileListing( storeDir, logFiles, labelScanStore, + indexingService, explicitIndexes, storageEngine ); ResourceIterator scanSnapshot = scanStoreFilesAre( labelScanStore, new String[]{"blah/scan.store", "scan.more"} ); @@ -168,6 +168,20 @@ public void shouldListMetaDataStoreLastWithTxLogs() throws Exception .isPresent() ); } + @Test + public void shouldListTransactionLogsFromCustomLocationWhenConfigured() throws IOException + { + String logFilesPath = "customTxFolder"; + verifyLogFilesWithCustomPathListing( logFilesPath ); + } + + @Test + public void shouldListTransactionLogsFromCustomAbsoluteLocationWhenConfigured() throws IOException + { + File customLogLocation = testDirectory.directory( "customLogLocation" ); + verifyLogFilesWithCustomPathListing( customLogLocation.getAbsolutePath() ); + } + @Test public void shouldListTxLogFiles() throws Exception { @@ -198,6 +212,20 @@ public void shouldListNeostoreFiles() throws Exception assertEquals( Arrays.asList(values), listedStoreFiles ); } + private void verifyLogFilesWithCustomPathListing( String path ) throws IOException + { + GraphDatabaseAPI graphDatabase = (GraphDatabaseAPI) new TestGraphDatabaseFactory() + .newEmbeddedDatabaseBuilder( testDirectory.directory( "customDb" ) ) + .setConfig( GraphDatabaseSettings.logical_logs_location, path ) + .newGraphDatabase(); + NeoStoreDataSource dataSource = graphDatabase.getDependencyResolver().resolveDependency( NeoStoreDataSource.class ); + LogFiles logFiles = graphDatabase.getDependencyResolver().resolveDependency( LogFiles.class ); + assertTrue( dataSource.listStoreFiles( true ).stream() + .anyMatch( metadata -> metadata.isLogFile() && logFiles.isLogFile( metadata.file() ) ) ); + assertEquals( Paths.get( path ).getFileName().toString(), logFiles.logFilesDirectory().getName() ); + graphDatabase.shutdown(); + } + private void filesInStoreDirAre( File storeDir, String[] filenames, String[] dirs ) { ArrayList files = new ArrayList<>(); @@ -226,6 +254,16 @@ private ResourceIterator indexFilesAre( IndexingService indexingService, S return snapshot; } + private void createIndexDbFile() throws IOException + { + File storeDir = db.getStoreDir(); + final File indexFile = new File( storeDir, "index.db" ); + if ( !indexFile.exists() ) + { + assertTrue( indexFile.createNewFile() ); + } + } + private void mockFiles( String[] filenames, ArrayList files, boolean isDirectories ) { for ( String filename : filenames ) diff --git a/community/kernel/src/test/java/org/neo4j/test/AdversarialPageCacheGraphDatabaseFactory.java b/community/kernel/src/test/java/org/neo4j/test/AdversarialPageCacheGraphDatabaseFactory.java index 1ec92f82405d3..f00c9dfebbbe0 100644 --- a/community/kernel/src/test/java/org/neo4j/test/AdversarialPageCacheGraphDatabaseFactory.java +++ b/community/kernel/src/test/java/org/neo4j/test/AdversarialPageCacheGraphDatabaseFactory.java @@ -20,12 +20,12 @@ package org.neo4j.test; import java.io.File; -import java.util.Map; import org.neo4j.adversaries.Adversary; import org.neo4j.adversaries.pagecache.AdversarialPageCache; import org.neo4j.graphdb.GraphDatabaseService; import org.neo4j.graphdb.factory.GraphDatabaseFactory; +import org.neo4j.graphdb.factory.GraphDatabaseSettings; import org.neo4j.io.fs.FileSystemAbstraction; import org.neo4j.io.pagecache.PageCache; import org.neo4j.kernel.configuration.Config; @@ -61,6 +61,7 @@ protected GraphDatabaseService newEmbeddedDatabase( File dir, Config config, Dep protected PlatformModule createPlatform( File storeDir, Config config, Dependencies dependencies, GraphDatabaseFacade facade ) { + config.augment( GraphDatabaseSettings.database_path, storeDir.getAbsolutePath() ); return new PlatformModule( storeDir, config, databaseInfo, dependencies, facade ) { @Override diff --git a/community/kernel/src/test/java/org/neo4j/test/TestGraphDatabaseFactory.java b/community/kernel/src/test/java/org/neo4j/test/TestGraphDatabaseFactory.java index 96143e53ce854..6b2afa348d5ed 100644 --- a/community/kernel/src/test/java/org/neo4j/test/TestGraphDatabaseFactory.java +++ b/community/kernel/src/test/java/org/neo4j/test/TestGraphDatabaseFactory.java @@ -259,6 +259,7 @@ protected TestGraphDatabaseFacadeFactory( TestGraphDatabaseFactoryState state, b protected PlatformModule createPlatform( File storeDir, Config config, Dependencies dependencies, GraphDatabaseFacade graphDatabaseFacade ) { + config.augment( GraphDatabaseSettings.database_path, storeDir.getAbsolutePath() ); if ( impermanent ) { config.augment( ephemeral, TRUE ); diff --git a/community/neo4j-harness/src/main/java/org/neo4j/harness/internal/AbstractInProcessServerBuilder.java b/community/neo4j-harness/src/main/java/org/neo4j/harness/internal/AbstractInProcessServerBuilder.java index 27d7d6b85a3d5..513fc6940fd97 100644 --- a/community/neo4j-harness/src/main/java/org/neo4j/harness/internal/AbstractInProcessServerBuilder.java +++ b/community/neo4j-harness/src/main/java/org/neo4j/harness/internal/AbstractInProcessServerBuilder.java @@ -54,8 +54,8 @@ import org.neo4j.server.configuration.ServerSettings; import org.neo4j.server.configuration.ThirdPartyJaxRsPackage; -import static org.neo4j.dbms.DatabaseManagementSystemSettings.data_directory; import static org.neo4j.graphdb.factory.GraphDatabaseSettings.auth_enabled; +import static org.neo4j.graphdb.factory.GraphDatabaseSettings.data_directory; import static org.neo4j.graphdb.factory.GraphDatabaseSettings.pagecache_memory; import static org.neo4j.helpers.collection.Iterables.append; import static org.neo4j.io.file.Files.createOrOpenAsOuputStream; diff --git a/community/neo4j-harness/src/test/java/org/neo4j/harness/InProcessBuilderTestIT.java b/community/neo4j-harness/src/test/java/org/neo4j/harness/InProcessBuilderTestIT.java index d0bf0c6bbfd35..15ec2c30f3ab5 100644 --- a/community/neo4j-harness/src/test/java/org/neo4j/harness/InProcessBuilderTestIT.java +++ b/community/neo4j-harness/src/test/java/org/neo4j/harness/InProcessBuilderTestIT.java @@ -42,7 +42,6 @@ import javax.net.ssl.X509TrustManager; import org.neo4j.bolt.v1.transport.socket.client.SocketConnection; -import org.neo4j.dbms.DatabaseManagementSystemSettings; import org.neo4j.graphdb.GraphDatabaseService; import org.neo4j.graphdb.Node; import org.neo4j.graphdb.ResourceIterable; @@ -179,8 +178,8 @@ public void shouldRunBuilderOnExistingStoreDir() throws Exception // When // create graph db with one node upfront Path dir = Files.createTempDirectory( getClass().getSimpleName() + "_shouldRunBuilderOnExistingStorageDir" ); - File storeDir = Config.defaults( DatabaseManagementSystemSettings.data_directory, dir.toString() ) - .get( DatabaseManagementSystemSettings.database_path ); + File storeDir = Config.defaults( GraphDatabaseSettings.data_directory, dir.toString() ) + .get( GraphDatabaseSettings.database_path ); try { GraphDatabaseService db = new TestGraphDatabaseFactory().newEmbeddedDatabase( storeDir ); diff --git a/community/server/src/main/java/org/neo4j/server/CommunityNeoServer.java b/community/server/src/main/java/org/neo4j/server/CommunityNeoServer.java index 3ed4a087a0993..8d6179e39cd19 100644 --- a/community/server/src/main/java/org/neo4j/server/CommunityNeoServer.java +++ b/community/server/src/main/java/org/neo4j/server/CommunityNeoServer.java @@ -24,7 +24,7 @@ import java.util.Arrays; import java.util.List; -import org.neo4j.dbms.DatabaseManagementSystemSettings; +import org.neo4j.graphdb.factory.GraphDatabaseSettings; import org.neo4j.kernel.configuration.Config; import org.neo4j.kernel.impl.factory.CommunityEditionModule; import org.neo4j.kernel.impl.factory.DatabaseInfo; @@ -53,7 +53,7 @@ public class CommunityNeoServer extends AbstractNeoServer { protected static final GraphFactory COMMUNITY_FACTORY = ( config, dependencies ) -> { - File storeDir = config.get( DatabaseManagementSystemSettings.database_path ); + File storeDir = config.get( GraphDatabaseSettings.database_path ); return new GraphDatabaseFacadeFactory( DatabaseInfo.COMMUNITY, CommunityEditionModule::new ) .newFacade( storeDir, config, dependencies ); }; diff --git a/community/server/src/main/java/org/neo4j/server/database/LifecycleManagingDatabase.java b/community/server/src/main/java/org/neo4j/server/database/LifecycleManagingDatabase.java index 712ace68d13b9..d7596b350559c 100644 --- a/community/server/src/main/java/org/neo4j/server/database/LifecycleManagingDatabase.java +++ b/community/server/src/main/java/org/neo4j/server/database/LifecycleManagingDatabase.java @@ -21,8 +21,8 @@ import java.io.File; -import org.neo4j.dbms.DatabaseManagementSystemSettings; import org.neo4j.graphdb.Result; +import org.neo4j.graphdb.factory.GraphDatabaseSettings; import org.neo4j.kernel.configuration.Config; import org.neo4j.kernel.impl.factory.GraphDatabaseFacade; import org.neo4j.kernel.impl.factory.GraphDatabaseFacadeFactory; @@ -67,7 +67,7 @@ public LifecycleManagingDatabase( Config config, GraphFactory dbFactory, @Override public File getLocation() { - return config.get( DatabaseManagementSystemSettings.database_path ); + return config.get( GraphDatabaseSettings.database_path ); } @Override diff --git a/community/server/src/test/java/org/neo4j/server/BaseBootstrapperTestIT.java b/community/server/src/test/java/org/neo4j/server/BaseBootstrapperTestIT.java index 526738b26fbc0..a601a4ff9eb8d 100644 --- a/community/server/src/test/java/org/neo4j/server/BaseBootstrapperTestIT.java +++ b/community/server/src/test/java/org/neo4j/server/BaseBootstrapperTestIT.java @@ -41,7 +41,7 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertThat; import static org.neo4j.bolt.v1.transport.integration.Neo4jWithSocket.DEFAULT_CONNECTOR_KEY; -import static org.neo4j.dbms.DatabaseManagementSystemSettings.data_directory; +import static org.neo4j.graphdb.factory.GraphDatabaseSettings.data_directory; import static org.neo4j.graphdb.factory.GraphDatabaseSettings.forced_kernel_id; import static org.neo4j.graphdb.factory.GraphDatabaseSettings.logs_directory; import static org.neo4j.helpers.collection.MapUtil.store; diff --git a/community/server/src/test/java/org/neo4j/server/ServerBootstrapperTest.java b/community/server/src/test/java/org/neo4j/server/ServerBootstrapperTest.java index 6b6cb2f7f0c61..0dee20f456f93 100644 --- a/community/server/src/test/java/org/neo4j/server/ServerBootstrapperTest.java +++ b/community/server/src/test/java/org/neo4j/server/ServerBootstrapperTest.java @@ -27,7 +27,6 @@ import java.util.Collection; import java.util.Collections; import java.util.Optional; - import javax.annotation.Nonnull; import org.neo4j.helpers.collection.MapUtil; @@ -42,8 +41,7 @@ import static org.hamcrest.Matchers.not; import static org.junit.Assert.assertThat; import static org.mockito.Mockito.mock; - -import static org.neo4j.dbms.DatabaseManagementSystemSettings.database_path; +import static org.neo4j.graphdb.factory.GraphDatabaseSettings.database_path; public class ServerBootstrapperTest { diff --git a/community/server/src/test/java/org/neo4j/server/ServerTestUtils.java b/community/server/src/test/java/org/neo4j/server/ServerTestUtils.java index 48047e9d98b85..569a8d95ebfe8 100644 --- a/community/server/src/test/java/org/neo4j/server/ServerTestUtils.java +++ b/community/server/src/test/java/org/neo4j/server/ServerTestUtils.java @@ -33,7 +33,6 @@ import java.util.Properties; import java.util.Random; -import org.neo4j.dbms.DatabaseManagementSystemSettings; import org.neo4j.graphdb.config.Setting; import org.neo4j.graphdb.factory.GraphDatabaseSettings; import org.neo4j.kernel.configuration.ssl.LegacySslPolicyConfig; @@ -90,7 +89,7 @@ public static Map getDefaultRelativeProperties() throws IOExcepti public static void addDefaultRelativeProperties( Map properties, File temporaryFolder ) { - addRelativeProperty( temporaryFolder, properties, DatabaseManagementSystemSettings.data_directory ); + addRelativeProperty( temporaryFolder, properties, GraphDatabaseSettings.data_directory ); addRelativeProperty( temporaryFolder, properties, GraphDatabaseSettings.logs_directory ); addRelativeProperty( temporaryFolder, properties, LegacySslPolicyConfig.certificates_directory ); properties.put( GraphDatabaseSettings.pagecache_memory.name(), "8m" ); diff --git a/community/server/src/test/java/org/neo4j/server/configuration/ConfigFileBuilder.java b/community/server/src/test/java/org/neo4j/server/configuration/ConfigFileBuilder.java index 412781451f01b..9c870b94b8bc8 100644 --- a/community/server/src/test/java/org/neo4j/server/configuration/ConfigFileBuilder.java +++ b/community/server/src/test/java/org/neo4j/server/configuration/ConfigFileBuilder.java @@ -23,8 +23,8 @@ import java.io.IOException; import java.util.Map; -import org.neo4j.dbms.DatabaseManagementSystemSettings; import org.neo4j.graphdb.config.Setting; +import org.neo4j.graphdb.factory.GraphDatabaseSettings; import org.neo4j.helpers.collection.MapUtil; import org.neo4j.server.ServerTestUtils; @@ -45,7 +45,7 @@ private ConfigFileBuilder( File directory ) //initialize config with defaults that doesn't pollute //workspace with generated data this.config = MapUtil.stringMap( - DatabaseManagementSystemSettings.data_directory.name(), directory.getAbsolutePath() + "/data", + GraphDatabaseSettings.data_directory.name(), directory.getAbsolutePath() + "/data", ServerSettings.management_api_path.name(), "http://localhost:7474/db/manage/", ServerSettings.rest_api_path.name(), "http://localhost:7474/db/data/" ); } diff --git a/community/server/src/test/java/org/neo4j/server/configuration/ConfigLoaderTest.java b/community/server/src/test/java/org/neo4j/server/configuration/ConfigLoaderTest.java index 1698b54fe3f91..9438319f35446 100644 --- a/community/server/src/test/java/org/neo4j/server/configuration/ConfigLoaderTest.java +++ b/community/server/src/test/java/org/neo4j/server/configuration/ConfigLoaderTest.java @@ -222,7 +222,7 @@ public void shouldDefaultToCorrectValueForAuthStoreLocation() throws IOException { File configFile = ConfigFileBuilder .builder( folder.getRoot() ) - .withoutSetting( DatabaseManagementSystemSettings.data_directory ) + .withoutSetting( GraphDatabaseSettings.data_directory ) .build(); Config config = Config.fromFile( configFile ).withHome( folder.getRoot() ).build(); @@ -234,7 +234,7 @@ public void shouldDefaultToCorrectValueForAuthStoreLocation() throws IOException public void shouldSetAValueForAuthStoreLocation() throws IOException { File configFile = ConfigFileBuilder.builder( folder.getRoot() ) - .withSetting( DatabaseManagementSystemSettings.data_directory, "the-data-dir" ) + .withSetting( GraphDatabaseSettings.data_directory, "the-data-dir" ) .build(); Config config = Config.fromFile( configFile ).withHome( folder.getRoot() ).build(); @@ -246,7 +246,7 @@ public void shouldSetAValueForAuthStoreLocation() throws IOException public void shouldNotOverwriteAuthStoreLocationIfProvided() throws IOException { File configFile = ConfigFileBuilder.builder( folder.getRoot() ) - .withSetting( DatabaseManagementSystemSettings.data_directory, "the-data-dir" ) + .withSetting( GraphDatabaseSettings.data_directory, "the-data-dir" ) .withSetting( GraphDatabaseSettings.auth_store, "foo/bar/auth" ) .build(); Config config = Config.fromFile( configFile ).withHome( folder.getRoot() ).build(); diff --git a/community/server/src/test/java/org/neo4j/server/database/TestLifecycleManagedDatabase.java b/community/server/src/test/java/org/neo4j/server/database/TestLifecycleManagedDatabase.java index f1bcf7f97dc57..255499b4b75bc 100644 --- a/community/server/src/test/java/org/neo4j/server/database/TestLifecycleManagedDatabase.java +++ b/community/server/src/test/java/org/neo4j/server/database/TestLifecycleManagedDatabase.java @@ -28,7 +28,7 @@ import java.io.File; import java.io.IOException; -import org.neo4j.dbms.DatabaseManagementSystemSettings; +import org.neo4j.graphdb.factory.GraphDatabaseSettings; import org.neo4j.kernel.GraphDatabaseDependencies; import org.neo4j.kernel.StoreLockException; import org.neo4j.kernel.configuration.Config; @@ -66,7 +66,7 @@ public void setup() throws Exception dataDirectory = createTempDir(); dbFactory = createGraphFactory(); - dbConfig = Config.defaults( DatabaseManagementSystemSettings.data_directory, dataDirectory.getAbsolutePath() ); + dbConfig = Config.defaults( GraphDatabaseSettings.data_directory, dataDirectory.getAbsolutePath() ); theDatabase = newDatabase(); } @@ -142,7 +142,7 @@ public void shouldBeAbleToGetLocation() throws Throwable { theDatabase.start(); assertThat( theDatabase.getLocation().getAbsolutePath(), - is( dbConfig.get( DatabaseManagementSystemSettings.database_path ).getAbsolutePath() ) ); + is( dbConfig.get( GraphDatabaseSettings.database_path ).getAbsolutePath() ) ); } private LifecycleManagingDatabase.GraphFactory createGraphFactory() diff --git a/community/server/src/test/java/org/neo4j/server/helpers/CommunityServerBuilder.java b/community/server/src/test/java/org/neo4j/server/helpers/CommunityServerBuilder.java index f9c4debc4e91f..2f3cb007f8265 100644 --- a/community/server/src/test/java/org/neo4j/server/helpers/CommunityServerBuilder.java +++ b/community/server/src/test/java/org/neo4j/server/helpers/CommunityServerBuilder.java @@ -28,7 +28,6 @@ import java.util.Map; import java.util.Properties; -import org.neo4j.dbms.DatabaseManagementSystemSettings; import org.neo4j.graphdb.factory.GraphDatabaseSettings; import org.neo4j.helpers.ListenSocketAddress; import org.neo4j.kernel.GraphDatabaseDependencies; @@ -79,7 +78,7 @@ public class CommunityServerBuilder private static LifecycleManagingDatabase.GraphFactory IN_MEMORY_DB = ( config, dependencies ) -> { - File storeDir = config.get( DatabaseManagementSystemSettings.database_path ); + File storeDir = config.get( GraphDatabaseSettings.database_path ); config.augment( stringMap( GraphDatabaseFacadeFactory.Configuration.ephemeral.name(), "true", new BoltConnector( "bolt" ).listen_address.name(), "localhost:0" ) ); return new ImpermanentGraphDatabase( storeDir, config, @@ -155,7 +154,7 @@ public Map createConfiguration( File temporaryFolder ) if ( dataDir != null ) { - properties.put( DatabaseManagementSystemSettings.data_directory.name(), dataDir ); + properties.put( GraphDatabaseSettings.data_directory.name(), dataDir ); } if ( maxThreads != null ) @@ -206,6 +205,8 @@ public Map createConfiguration( File temporaryFolder ) new File( temporaryFolder, "certificates" ).getAbsolutePath() ); properties.put( GraphDatabaseSettings.logs_directory.name(), new File( temporaryFolder, "logs" ).getAbsolutePath() ); + properties.put( GraphDatabaseSettings.logical_logs_location.name(), + new File( temporaryFolder, "transaction-logs" ).getAbsolutePath() ); properties.put( GraphDatabaseSettings.pagecache_memory.name(), "8m" ); for ( Object key : arbitraryProperties.keySet() ) diff --git a/community/server/src/test/java/org/neo4j/server/integration/StartupLoggingIT.java b/community/server/src/test/java/org/neo4j/server/integration/StartupLoggingIT.java index 62a8ea318698e..357bf43a61a0b 100644 --- a/community/server/src/test/java/org/neo4j/server/integration/StartupLoggingIT.java +++ b/community/server/src/test/java/org/neo4j/server/integration/StartupLoggingIT.java @@ -33,7 +33,6 @@ import java.util.Map; import java.util.Optional; -import org.neo4j.dbms.DatabaseManagementSystemSettings; import org.neo4j.graphdb.factory.GraphDatabaseSettings; import org.neo4j.io.fs.FileUtils; import org.neo4j.kernel.configuration.BoltConnector; @@ -47,10 +46,8 @@ import org.neo4j.test.rule.TestDirectory; import org.neo4j.test.server.ExclusiveServerTestBase; -import static org.hamcrest.MatcherAssert.assertThat; - import static java.util.Arrays.asList; - +import static org.hamcrest.MatcherAssert.assertThat; import static org.neo4j.bolt.v1.transport.integration.Neo4jWithSocket.DEFAULT_CONNECTOR_KEY; import static org.neo4j.server.AbstractNeoServer.NEO4J_IS_STARTING_MESSAGE; @@ -62,7 +59,7 @@ public class StartupLoggingIT extends ExclusiveServerTestBase @Before public void setUp() throws IOException { - FileUtils.deleteRecursively( ServerTestUtils.getRelativeFile( DatabaseManagementSystemSettings.data_directory ) ); + FileUtils.deleteRecursively( ServerTestUtils.getRelativeFile( GraphDatabaseSettings.data_directory ) ); } @Rule @@ -111,7 +108,7 @@ private Map getPropertyPairs() throws IOException relativeProperties.put( bolt.enabled.name(), "true" ); relativeProperties.put( bolt.listen_address.name(), "localhost:" + PortAuthority.allocatePort() ); - relativeProperties.put( DatabaseManagementSystemSettings.database_path.name(), + relativeProperties.put( GraphDatabaseSettings.database_path.name(), homeDir.absolutePath().getAbsolutePath() ); return relativeProperties; } diff --git a/enterprise/backup/src/main/java/org/neo4j/backup/AbstractBackupSupportingClassesFactory.java b/enterprise/backup/src/main/java/org/neo4j/backup/AbstractBackupSupportingClassesFactory.java index 431ae532aea5c..488b6b9dee6c8 100644 --- a/enterprise/backup/src/main/java/org/neo4j/backup/AbstractBackupSupportingClassesFactory.java +++ b/enterprise/backup/src/main/java/org/neo4j/backup/AbstractBackupSupportingClassesFactory.java @@ -29,13 +29,10 @@ import org.neo4j.causalclustering.catchup.tx.TxPullClient; import org.neo4j.causalclustering.handlers.PipelineHandlerAppender; import org.neo4j.causalclustering.handlers.PipelineHandlerAppenderFactory; -import org.neo4j.graphdb.DependencyResolver; import org.neo4j.io.fs.FileSystemAbstraction; import org.neo4j.io.pagecache.PageCache; import org.neo4j.kernel.configuration.Config; -import org.neo4j.kernel.configuration.ssl.SslPolicyLoader; import org.neo4j.kernel.impl.pagecache.ConfigurableStandalonePageCacheFactory; -import org.neo4j.kernel.impl.transaction.state.DataSourceManager; import org.neo4j.kernel.impl.util.Dependencies; import org.neo4j.kernel.monitoring.Monitors; import org.neo4j.logging.LogProvider; @@ -94,8 +91,8 @@ private BackupDelegator backupDelegatorFromConfig( PageCache pageCache, Config c TxPullClient txPullClient = new TxPullClient( catchUpClient, monitors ); StoreCopyClient storeCopyClient = new StoreCopyClient( catchUpClient, logProvider ); - RemoteStore remoteStore = - new RemoteStore( logProvider, fileSystemAbstraction, pageCache, storeCopyClient, txPullClient, transactionLogCatchUpFactory, monitors ); + RemoteStore remoteStore = new RemoteStore( logProvider, fileSystemAbstraction, pageCache, storeCopyClient, + txPullClient, transactionLogCatchUpFactory, config, monitors ); return backupDelegator( remoteStore, catchUpClient, storeCopyClient ); } diff --git a/enterprise/backup/src/main/java/org/neo4j/backup/BackupDelegator.java b/enterprise/backup/src/main/java/org/neo4j/backup/BackupDelegator.java index ecce952610af8..73a98fa0b3d9e 100644 --- a/enterprise/backup/src/main/java/org/neo4j/backup/BackupDelegator.java +++ b/enterprise/backup/src/main/java/org/neo4j/backup/BackupDelegator.java @@ -70,7 +70,7 @@ CatchupResult tryCatchingUp( AdvertisedSocketAddress fromAddress, StoreId expect { try { - return remoteStore.tryCatchingUp( fromAddress, expectedStoreId, storeDir ); + return remoteStore.tryCatchingUp( fromAddress, expectedStoreId, storeDir, true ); } catch ( IOException e ) { diff --git a/enterprise/backup/src/main/java/org/neo4j/backup/BackupProtocolService.java b/enterprise/backup/src/main/java/org/neo4j/backup/BackupProtocolService.java index 5994ba2ac66c2..efed81c97acae 100644 --- a/enterprise/backup/src/main/java/org/neo4j/backup/BackupProtocolService.java +++ b/enterprise/backup/src/main/java/org/neo4j/backup/BackupProtocolService.java @@ -27,7 +27,6 @@ import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.Optional; import java.util.function.Supplier; import javax.annotation.Nullable; @@ -80,7 +79,6 @@ import static org.neo4j.com.storecopy.TransactionCommittingResponseUnpacker.DEFAULT_BATCH_SIZE; import static org.neo4j.graphdb.factory.GraphDatabaseSettings.logs_directory; import static org.neo4j.graphdb.factory.GraphDatabaseSettings.store_internal_log_path; -import static org.neo4j.helpers.Exceptions.launderedException; import static org.neo4j.helpers.Exceptions.rootCause; import static org.neo4j.kernel.impl.pagecache.ConfigurableStandalonePageCacheFactory.createPageCache; diff --git a/enterprise/backup/src/main/java/org/neo4j/backup/BackupRecoveryService.java b/enterprise/backup/src/main/java/org/neo4j/backup/BackupRecoveryService.java index 5c20af3748f59..ffaf6379c12fd 100644 --- a/enterprise/backup/src/main/java/org/neo4j/backup/BackupRecoveryService.java +++ b/enterprise/backup/src/main/java/org/neo4j/backup/BackupRecoveryService.java @@ -22,6 +22,7 @@ import java.io.File; import java.util.Map; +import org.neo4j.graphdb.factory.GraphDatabaseSettings; import org.neo4j.io.pagecache.PageCache; import org.neo4j.kernel.configuration.Config; import org.neo4j.kernel.internal.GraphDatabaseAPI; @@ -33,6 +34,7 @@ public class BackupRecoveryService public void recoverWithDatabase( File targetDirectory, PageCache pageCache, Config config ) { Map configParams = config.getRaw(); + configParams.put( GraphDatabaseSettings.logical_logs_location.name(), targetDirectory.getAbsolutePath() ); GraphDatabaseAPI targetDb = startTemporaryDb( targetDirectory, pageCache, configParams ); targetDb.shutdown(); } diff --git a/enterprise/backup/src/main/java/org/neo4j/backup/BackupStrategyWrapper.java b/enterprise/backup/src/main/java/org/neo4j/backup/BackupStrategyWrapper.java index 3bb8c11d98813..6495b32fbf6f1 100644 --- a/enterprise/backup/src/main/java/org/neo4j/backup/BackupStrategyWrapper.java +++ b/enterprise/backup/src/main/java/org/neo4j/backup/BackupStrategyWrapper.java @@ -20,17 +20,13 @@ package org.neo4j.backup; import java.io.File; -import java.util.Map; import org.neo4j.commandline.admin.CommandFailed; import org.neo4j.helpers.OptionalHostnamePort; import org.neo4j.io.pagecache.PageCache; import org.neo4j.kernel.configuration.Config; -import org.neo4j.kernel.internal.GraphDatabaseAPI; import org.neo4j.kernel.lifecycle.LifeSupport; -import static org.neo4j.backup.BackupProtocolService.startTemporaryDb; - class BackupStrategyWrapper { private final BackupStrategy backupStrategy; diff --git a/enterprise/backup/src/main/java/org/neo4j/backup/OnlineBackupCommandConfigLoader.java b/enterprise/backup/src/main/java/org/neo4j/backup/OnlineBackupCommandConfigLoader.java index ebb6032eb6446..a1b80478f06ef 100644 --- a/enterprise/backup/src/main/java/org/neo4j/backup/OnlineBackupCommandConfigLoader.java +++ b/enterprise/backup/src/main/java/org/neo4j/backup/OnlineBackupCommandConfigLoader.java @@ -40,8 +40,9 @@ class OnlineBackupCommandConfigLoader Config loadConfig( Optional additionalConfig ) throws CommandFailed { - return withAdditionalConfig( additionalConfig, - Config.fromFile( configDir.resolve( Config.DEFAULT_CONFIG_FILE_NAME ) ).withHome( homeDir ).withConnectorsDisabled().build() ); + Config config = Config.fromFile( configDir.resolve( Config.DEFAULT_CONFIG_FILE_NAME ) ).withHome( homeDir ) + .withConnectorsDisabled().build(); + return withAdditionalConfig( additionalConfig, config ); } private Config withAdditionalConfig( Optional additionalConfig, Config config ) throws CommandFailed diff --git a/enterprise/backup/src/main/java/org/neo4j/backup/OnlineBackupRequiredArguments.java b/enterprise/backup/src/main/java/org/neo4j/backup/OnlineBackupRequiredArguments.java index 3b99ec576c95c..6c0d287fa0caa 100644 --- a/enterprise/backup/src/main/java/org/neo4j/backup/OnlineBackupRequiredArguments.java +++ b/enterprise/backup/src/main/java/org/neo4j/backup/OnlineBackupRequiredArguments.java @@ -36,8 +36,7 @@ public class OnlineBackupRequiredArguments private final Path reportDir; public OnlineBackupRequiredArguments( OptionalHostnamePort address, Path folder, String name, boolean fallbackToFull, boolean doConsistencyCheck, - long timeout, - Optional additionalConfig, Path reportDir ) + long timeout, Optional additionalConfig, Path reportDir ) { this.address = address; this.folder = folder; diff --git a/enterprise/backup/src/main/java/org/neo4j/restore/RestoreDatabaseCli.java b/enterprise/backup/src/main/java/org/neo4j/restore/RestoreDatabaseCli.java index a57bf1fff5150..9f59bcbb58fc7 100644 --- a/enterprise/backup/src/main/java/org/neo4j/restore/RestoreDatabaseCli.java +++ b/enterprise/backup/src/main/java/org/neo4j/restore/RestoreDatabaseCli.java @@ -29,7 +29,7 @@ import org.neo4j.commandline.arguments.Arguments; import org.neo4j.commandline.arguments.MandatoryNamedArg; import org.neo4j.commandline.arguments.OptionalBooleanArg; -import org.neo4j.dbms.DatabaseManagementSystemSettings; +import org.neo4j.graphdb.factory.GraphDatabaseSettings; import org.neo4j.io.fs.DefaultFileSystemAbstraction; import org.neo4j.io.fs.FileSystemAbstraction; import org.neo4j.kernel.configuration.Config; @@ -53,7 +53,7 @@ private static Config loadNeo4jConfig( Path homeDir, Path configDir, String data { return Config.fromFile( configDir.resolve( Config.DEFAULT_CONFIG_FILE_NAME ) ) .withHome( homeDir ) - .withSetting( DatabaseManagementSystemSettings.active_database, databaseName ) + .withSetting( GraphDatabaseSettings.active_database, databaseName ) .withConnectorsDisabled().build(); } diff --git a/enterprise/backup/src/main/java/org/neo4j/restore/RestoreDatabaseCommand.java b/enterprise/backup/src/main/java/org/neo4j/restore/RestoreDatabaseCommand.java index 4e0c35596bd87..50cb4808bb4e8 100644 --- a/enterprise/backup/src/main/java/org/neo4j/restore/RestoreDatabaseCommand.java +++ b/enterprise/backup/src/main/java/org/neo4j/restore/RestoreDatabaseCommand.java @@ -23,30 +23,36 @@ import java.io.IOException; import org.neo4j.commandline.admin.CommandFailed; +import org.neo4j.graphdb.factory.GraphDatabaseSettings; import org.neo4j.io.fs.FileSystemAbstraction; import org.neo4j.kernel.configuration.Config; +import org.neo4j.kernel.impl.transaction.log.files.LogFiles; +import org.neo4j.kernel.impl.transaction.log.files.LogFilesBuilder; import org.neo4j.kernel.impl.util.Validators; import static java.lang.String.format; import static org.neo4j.commandline.Util.checkLock; -import static org.neo4j.dbms.DatabaseManagementSystemSettings.database_path; +import static org.neo4j.commandline.Util.isSameOrChildFile; +import static org.neo4j.graphdb.factory.GraphDatabaseSettings.database_path; public class RestoreDatabaseCommand { private FileSystemAbstraction fs; private final File fromPath; private final File databaseDir; + private final File transactionLogsDirectory; private String databaseName; private boolean forceOverwrite; - public RestoreDatabaseCommand( FileSystemAbstraction fs, File fromPath, Config config, - String databaseName, boolean forceOverwrite ) + public RestoreDatabaseCommand( FileSystemAbstraction fs, File fromPath, Config config, String databaseName, + boolean forceOverwrite ) { this.fs = fs; this.fromPath = fromPath; this.databaseName = databaseName; this.forceOverwrite = forceOverwrite; this.databaseDir = config.get( database_path ).getAbsoluteFile(); + this.transactionLogsDirectory = config.get( GraphDatabaseSettings.logical_logs_location ).getAbsoluteFile(); } public void execute() throws IOException, CommandFailed @@ -75,6 +81,19 @@ public void execute() throws IOException, CommandFailed checkLock( databaseDir.toPath() ); fs.deleteRecursively( databaseDir ); - fs.copyRecursively( fromPath, databaseDir ); + + if ( !isSameOrChildFile( databaseDir, transactionLogsDirectory ) ) + { + fs.deleteRecursively( transactionLogsDirectory ); + } + LogFiles backupLogFiles = LogFilesBuilder.logFilesBasedOnlyBuilder( fromPath, fs ).build(); + File[] files = fromPath.listFiles(); + if ( files != null ) + { + for ( File file : files ) + { + fs.copyToDirectory( file, backupLogFiles.isLogFile( file ) ? transactionLogsDirectory : databaseDir ); + } + } } } diff --git a/enterprise/backup/src/test/java/org/neo4j/backup/BackupDelegatorTest.java b/enterprise/backup/src/test/java/org/neo4j/backup/BackupDelegatorTest.java index 54004dc8f2c5a..c5e410c77b370 100644 --- a/enterprise/backup/src/test/java/org/neo4j/backup/BackupDelegatorTest.java +++ b/enterprise/backup/src/test/java/org/neo4j/backup/BackupDelegatorTest.java @@ -73,7 +73,7 @@ public void tryCatchingUpDelegatesToRemoteStore() throws org.neo4j.causalcluster subject.tryCatchingUp( fromAddress, expectedStoreId, storeDir ); // then - verify( remoteStore ).tryCatchingUp( fromAddress, expectedStoreId, storeDir ); + verify( remoteStore ).tryCatchingUp( fromAddress, expectedStoreId, storeDir, true ); } @Test diff --git a/enterprise/backup/src/test/java/org/neo4j/backup/BackupFlowTest.java b/enterprise/backup/src/test/java/org/neo4j/backup/BackupFlowTest.java index c3565908b09b9..8a013c24ec798 100644 --- a/enterprise/backup/src/test/java/org/neo4j/backup/BackupFlowTest.java +++ b/enterprise/backup/src/test/java/org/neo4j/backup/BackupFlowTest.java @@ -35,6 +35,7 @@ import org.neo4j.consistency.checking.full.ConsistencyCheckIncompleteException; import org.neo4j.consistency.checking.full.ConsistencyFlags; import org.neo4j.helpers.progress.ProgressMonitorFactory; +import org.neo4j.io.fs.FileSystemAbstraction; import org.neo4j.logging.LogProvider; import static org.hamcrest.CoreMatchers.equalTo; @@ -53,26 +54,28 @@ public class BackupFlowTest public ExpectedException expectedException = ExpectedException.none(); // dependencies - final ConsistencyCheckService consistencyCheckService = mock( ConsistencyCheckService.class ); - final OutsideWorld outsideWorld = mock( OutsideWorld.class ); - final LogProvider logProvider = mock( LogProvider.class ); - final BackupStrategyWrapper firstStrategy = mock( BackupStrategyWrapper.class ); - final BackupStrategyWrapper secondStrategy = mock( BackupStrategyWrapper.class ); + private final ConsistencyCheckService consistencyCheckService = mock( ConsistencyCheckService.class ); + private final OutsideWorld outsideWorld = mock( OutsideWorld.class ); + private final FileSystemAbstraction fileSystem = mock( FileSystemAbstraction.class ); + private final LogProvider logProvider = mock( LogProvider.class ); + private final BackupStrategyWrapper firstStrategy = mock( BackupStrategyWrapper.class ); + private final BackupStrategyWrapper secondStrategy = mock( BackupStrategyWrapper.class ); BackupFlow subject; // test method parameter mocks - final OnlineBackupContext onlineBackupContext = mock( OnlineBackupContext.class ); - final OnlineBackupRequiredArguments requiredArguments = mock( OnlineBackupRequiredArguments.class ); + private final OnlineBackupContext onlineBackupContext = mock( OnlineBackupContext.class ); + private final OnlineBackupRequiredArguments requiredArguments = mock( OnlineBackupRequiredArguments.class ); // mock returns - private ProgressMonitorFactory progressMonitorFactory = mock( ProgressMonitorFactory.class ); - private Path reportDir = mock( Path.class ); - private ConsistencyCheckService.Result consistencyCheckResult = mock( ConsistencyCheckService.Result.class ); + private final ProgressMonitorFactory progressMonitorFactory = mock( ProgressMonitorFactory.class ); + private final Path reportDir = mock( Path.class ); + private final ConsistencyCheckService.Result consistencyCheckResult = mock( ConsistencyCheckService.Result.class ); @Before public void setup() { + when( outsideWorld.fileSystem() ).thenReturn( fileSystem ); when( onlineBackupContext.getRequiredArguments() ).thenReturn( requiredArguments ); when( requiredArguments.getReportDir() ).thenReturn( reportDir ); subject = new BackupFlow( consistencyCheckService, outsideWorld, logProvider, progressMonitorFactory, diff --git a/enterprise/backup/src/test/java/org/neo4j/backup/BackupIT.java b/enterprise/backup/src/test/java/org/neo4j/backup/BackupIT.java index bc5552589418e..7b0281236dac4 100644 --- a/enterprise/backup/src/test/java/org/neo4j/backup/BackupIT.java +++ b/enterprise/backup/src/test/java/org/neo4j/backup/BackupIT.java @@ -20,6 +20,7 @@ package org.neo4j.backup; import org.apache.commons.io.FileUtils; +import org.hamcrest.Matchers; import org.junit.After; import org.junit.Before; import org.junit.Rule; @@ -66,6 +67,8 @@ import org.neo4j.kernel.impl.store.id.IdGeneratorImpl; import org.neo4j.kernel.impl.storemigration.StoreFileType; import org.neo4j.kernel.impl.transaction.TransactionHeaderInformationFactory; +import org.neo4j.kernel.impl.transaction.log.files.LogFiles; +import org.neo4j.kernel.impl.transaction.log.files.LogFilesBuilder; import org.neo4j.ports.allocation.PortAuthority; import org.neo4j.test.DbRepresentation; import org.neo4j.test.TestGraphDatabaseFactory; @@ -493,6 +496,34 @@ public void shouldLeaveIdFilesAfterBackup() throws Exception } } + @Test + public void backupDatabaseWithCustomTransactionLogsLocation() throws IOException + { + int backupPort = PortAuthority.allocatePort(); + GraphDatabaseService db = startGraphDatabase( serverPath, true, backupPort, "customLogLocation" ); + try + { + createInitialDataset( db ); + + OnlineBackup backup = OnlineBackup.from( "127.0.0.1", backupPort ); + String backupStore = backupPath.getPath(); + LogFiles logFiles = LogFilesBuilder.logFilesBasedOnlyBuilder( new File( backupStore ), fileSystemRule.get() ).build(); + + backup.full( backupStore ); + assertThat( logFiles.logFiles(), Matchers.arrayWithSize( 1 ) ); + + DbRepresentation representation = addLotsOfData( db ); + backup.incremental( backupStore ); + assertThat( logFiles.logFiles(), Matchers.arrayWithSize( 1 ) ); + + assertEquals( representation, getDbRepresentation() ); + } + finally + { + db.shutdown(); + } + } + private void ensureStoresHaveIdFiles( File path ) throws IOException { for ( StoreFile file : StoreFile.values() ) @@ -637,6 +668,12 @@ private DbRepresentation addMoreData( File path ) } private GraphDatabaseService startGraphDatabase( File storeDir, boolean withOnlineBackup, Integer backupPort ) + { + return startGraphDatabase( storeDir, withOnlineBackup, backupPort, "" ); + } + + private GraphDatabaseService startGraphDatabase( File storeDir, boolean withOnlineBackup, Integer backupPort, + String logLocation ) { GraphDatabaseFactory dbFactory = new TestGraphDatabaseFactory() { @@ -668,7 +705,8 @@ protected TransactionHeaderInformation createUsing( byte[] additionalHeader ) GraphDatabaseBuilder graphDatabaseBuilder = dbFactory.newEmbeddedDatabaseBuilder( storeDir ) .setConfig( OnlineBackupSettings.online_backup_enabled, String.valueOf( withOnlineBackup ) ) .setConfig( GraphDatabaseSettings.keep_logical_logs, Settings.TRUE ) - .setConfig( GraphDatabaseSettings.record_format, recordFormatName ); + .setConfig( GraphDatabaseSettings.record_format, recordFormatName ) + .setConfig( GraphDatabaseSettings.logical_logs_location, logLocation ); if ( backupPort != null ) { diff --git a/enterprise/backup/src/test/java/org/neo4j/backup/BackupProtocolServiceIT.java b/enterprise/backup/src/test/java/org/neo4j/backup/BackupProtocolServiceIT.java index 0ae2d19fff55b..8f10f30c720c0 100644 --- a/enterprise/backup/src/test/java/org/neo4j/backup/BackupProtocolServiceIT.java +++ b/enterprise/backup/src/test/java/org/neo4j/backup/BackupProtocolServiceIT.java @@ -1087,7 +1087,7 @@ private void checkLastCommittedTxIdInLogAndNeoStore( long txId, long txIdFromOri LifeSupport life = new LifeSupport(); PageCache pageCache = pageCacheRule.getPageCache( fileSystem ); LogicalTransactionStore transactionStore = - life.add( new ReadOnlyTransactionStore( pageCache, fileSystem, backupDir, monitors ) ); + life.add( new ReadOnlyTransactionStore( pageCache, fileSystem, backupDir, Config.defaults(), monitors ) ); life.start(); try ( IOCursor cursor = transactionStore.getTransactions( txId ) ) diff --git a/enterprise/backup/src/test/java/org/neo4j/backup/OnlineBackupCommandConfigLoaderTest.java b/enterprise/backup/src/test/java/org/neo4j/backup/OnlineBackupCommandConfigLoaderTest.java index dfcdf7e1ca29d..62920e2d57039 100644 --- a/enterprise/backup/src/test/java/org/neo4j/backup/OnlineBackupCommandConfigLoaderTest.java +++ b/enterprise/backup/src/test/java/org/neo4j/backup/OnlineBackupCommandConfigLoaderTest.java @@ -23,7 +23,6 @@ import org.junit.Rule; import org.junit.Test; import org.junit.rules.ExpectedException; -import org.junit.rules.TemporaryFolder; import java.io.BufferedWriter; import java.io.File; @@ -35,7 +34,6 @@ import org.neo4j.causalclustering.core.CausalClusteringSettings; import org.neo4j.commandline.admin.CommandFailed; -import org.neo4j.graphdb.config.InvalidSettingException; import org.neo4j.kernel.configuration.Config; import org.neo4j.test.rule.TestDirectory; diff --git a/enterprise/backup/src/test/java/org/neo4j/causalclustering/BackupCoreIT.java b/enterprise/backup/src/test/java/org/neo4j/causalclustering/BackupCoreIT.java index ee1979c06769a..c755f095411a7 100644 --- a/enterprise/backup/src/test/java/org/neo4j/causalclustering/BackupCoreIT.java +++ b/enterprise/backup/src/test/java/org/neo4j/causalclustering/BackupCoreIT.java @@ -24,27 +24,24 @@ import org.junit.Test; import java.io.File; -import java.net.InetSocketAddress; import java.util.ArrayList; import java.util.List; import java.util.Map; -import org.neo4j.causalclustering.core.CausalClusteringSettings; import org.neo4j.causalclustering.core.CoreGraphDatabase; import org.neo4j.causalclustering.core.consensus.roles.Role; import org.neo4j.causalclustering.discovery.Cluster; import org.neo4j.causalclustering.discovery.CoreClusterMember; import org.neo4j.graphdb.Node; import org.neo4j.graphdb.factory.GraphDatabaseSettings; -import org.neo4j.helpers.HostnamePort; import org.neo4j.helpers.collection.MapUtil; import org.neo4j.kernel.configuration.Config; import org.neo4j.kernel.impl.enterprise.configuration.OnlineBackupSettings; -import org.neo4j.kernel.impl.factory.GraphDatabaseFacade; import org.neo4j.kernel.impl.store.format.standard.Standard; import org.neo4j.ports.allocation.PortAuthority; import org.neo4j.test.DbRepresentation; import org.neo4j.test.causalclustering.ClusterRule; +import org.neo4j.test.rule.SuppressOutput; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotEquals; @@ -54,7 +51,9 @@ public class BackupCoreIT { @Rule - public ClusterRule clusterRule = new ClusterRule( getClass() ) + public final SuppressOutput suppressOutput = SuppressOutput.suppressAll(); + @Rule + public final ClusterRule clusterRule = new ClusterRule( getClass() ) .withNumberOfCoreMembers( 3 ) .withNumberOfReadReplicas( 0 ); diff --git a/enterprise/backup/src/test/java/org/neo4j/causalclustering/ClusterSeedingIT.java b/enterprise/backup/src/test/java/org/neo4j/causalclustering/ClusterSeedingIT.java index 4c22b8c682f71..3c1f42ed64fdc 100644 --- a/enterprise/backup/src/test/java/org/neo4j/causalclustering/ClusterSeedingIT.java +++ b/enterprise/backup/src/test/java/org/neo4j/causalclustering/ClusterSeedingIT.java @@ -32,16 +32,17 @@ import org.neo4j.causalclustering.discovery.CoreClusterMember; import org.neo4j.causalclustering.discovery.IpFamily; import org.neo4j.causalclustering.discovery.SharedDiscoveryService; +import org.neo4j.graphdb.factory.GraphDatabaseSettings; import org.neo4j.io.fs.FileSystemAbstraction; import org.neo4j.kernel.configuration.Config; import org.neo4j.kernel.impl.enterprise.configuration.OnlineBackupSettings; import org.neo4j.kernel.impl.store.format.standard.Standard; +import org.neo4j.restore.RestoreDatabaseCommand; import org.neo4j.test.DbRepresentation; import org.neo4j.test.rule.TestDirectory; import org.neo4j.test.rule.fs.DefaultFileSystemRule; import static java.util.Collections.emptyMap; -import static java.util.Collections.singletonMap; import static org.junit.Assert.assertEquals; import static org.neo4j.causalclustering.BackupCoreIT.backupAddress; import static org.neo4j.causalclustering.discovery.Cluster.dataMatchesEventually; @@ -136,7 +137,8 @@ public void shouldSeedNewMemberFromEmptyIdleCluster() throws Throwable // and: seeding new member with said backup CoreClusterMember newMember = cluster.addCoreMemberWithId( 3 ); - fsa.copyRecursively( backupDir, newMember.storeDir() ); + String databaseName = newMember.getMemberConfig().get( GraphDatabaseSettings.active_database ); + new RestoreDatabaseCommand( fsa, backupDir, newMember.getMemberConfig(), databaseName, true ).execute(); newMember.start(); // then @@ -158,7 +160,8 @@ public void shouldSeedNewMemberFromNonEmptyIdleCluster() throws Throwable // and: seeding new member with said backup CoreClusterMember newMember = cluster.addCoreMemberWithId( 3 ); - fsa.copyRecursively( backupDir, newMember.storeDir() ); + String databaseName = newMember.getMemberConfig().get( GraphDatabaseSettings.active_database ); + new RestoreDatabaseCommand( fsa, backupDir, newMember.getMemberConfig(), databaseName, true ).execute(); newMember.start(); // then diff --git a/enterprise/backup/src/test/java/org/neo4j/restore/RestoreDatabaseCommandIT.java b/enterprise/backup/src/test/java/org/neo4j/restore/RestoreDatabaseCommandIT.java index 272f942873f12..dda5120b588d5 100644 --- a/enterprise/backup/src/test/java/org/neo4j/restore/RestoreDatabaseCommandIT.java +++ b/enterprise/backup/src/test/java/org/neo4j/restore/RestoreDatabaseCommandIT.java @@ -21,33 +21,42 @@ import org.junit.Rule; import org.junit.Test; +import org.mockito.Mockito; import java.io.ByteArrayOutputStream; import java.io.File; +import java.io.IOException; import java.io.PrintStream; +import org.neo4j.commandline.admin.CommandFailed; import org.neo4j.commandline.admin.CommandLocator; import org.neo4j.commandline.admin.Usage; -import org.neo4j.dbms.DatabaseManagementSystemSettings; import org.neo4j.graphdb.GraphDatabaseService; import org.neo4j.graphdb.Transaction; import org.neo4j.graphdb.factory.GraphDatabaseFactory; +import org.neo4j.graphdb.factory.GraphDatabaseSettings; import org.neo4j.helpers.collection.Iterables; import org.neo4j.io.fs.FileSystemAbstraction; import org.neo4j.kernel.configuration.Config; import org.neo4j.kernel.configuration.Settings; import org.neo4j.kernel.impl.enterprise.configuration.OnlineBackupSettings; +import org.neo4j.kernel.impl.transaction.log.files.LogFiles; +import org.neo4j.kernel.impl.transaction.log.files.LogFilesBuilder; import org.neo4j.kernel.internal.locker.StoreLocker; -import org.neo4j.ports.allocation.PortAuthority; import org.neo4j.test.rule.TestDirectory; import org.neo4j.test.rule.fs.DefaultFileSystemRule; +import static org.hamcrest.Matchers.arrayWithSize; +import static org.hamcrest.Matchers.emptyArray; import static org.hamcrest.Matchers.equalTo; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertThat; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; +import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; import static org.neo4j.helpers.collection.MapUtil.stringMap; public class RestoreDatabaseCommandIT @@ -64,7 +73,7 @@ public void forceShouldRespectStoreLock() throws Exception Config config = configWith( databaseName, directory.absolutePath().getAbsolutePath() ); File fromPath = new File( directory.absolutePath(), "from" ); - File toPath = config.get( DatabaseManagementSystemSettings.database_path ); + File toPath = config.get( GraphDatabaseSettings.database_path ); int fromNodeCount = 10; int toNodeCount = 20; @@ -93,7 +102,7 @@ public void shouldNotCopyOverAndExistingDatabase() throws Exception Config config = configWith( databaseName, directory.absolutePath().getAbsolutePath() ); File fromPath = new File( directory.absolutePath(), "from" ); - File toPath = config.get( DatabaseManagementSystemSettings.database_path ); + File toPath = config.get( GraphDatabaseSettings.database_path ); createDbAt( fromPath, 0 ); createDbAt( toPath, 0 ); @@ -121,7 +130,7 @@ public void shouldThrowExceptionIfBackupDirectoryDoesNotExist() throws Exception Config config = configWith( databaseName, directory.absolutePath().getAbsolutePath() ); File fromPath = new File( directory.absolutePath(), "from" ); - File toPath = config.get( DatabaseManagementSystemSettings.database_path ); + File toPath = config.get( GraphDatabaseSettings.database_path ); createDbAt( toPath, 0 ); @@ -171,7 +180,7 @@ public void shouldAllowForcedCopyOverAnExistingDatabase() throws Exception Config config = configWith( databaseName, directory.absolutePath().getAbsolutePath() ); File fromPath = new File( directory.absolutePath(), "from" ); - File toPath = config.get( DatabaseManagementSystemSettings.database_path ); + File toPath = config.get( GraphDatabaseSettings.database_path ); int fromNodeCount = 10; int toNodeCount = 20; @@ -194,6 +203,61 @@ public void shouldAllowForcedCopyOverAnExistingDatabase() throws Exception copiedDb.shutdown(); } + @Test + public void restoreTransactionLogsInCustomDirectoryForTargetDatabaseWhenConfigured() + throws IOException, CommandFailed + { + String databaseName = "to"; + Config config = configWith( databaseName, directory.absolutePath().getAbsolutePath() ); + File customTxLogDirectory = directory.directory( "customLogicalLog" ); + String customTransactionLogDirectory = customTxLogDirectory.getAbsolutePath(); + config.augmentDefaults( GraphDatabaseSettings.logical_logs_location, customTransactionLogDirectory ); + + File fromPath = new File( directory.absolutePath(), "from" ); + File toPath = config.get( GraphDatabaseSettings.database_path ); + int fromNodeCount = 10; + int toNodeCount = 20; + createDbAt( fromPath, fromNodeCount ); + + GraphDatabaseService db = new GraphDatabaseFactory() + .newEmbeddedDatabaseBuilder( toPath ) + .setConfig( OnlineBackupSettings.online_backup_enabled, Settings.FALSE ) + .setConfig( GraphDatabaseSettings.logical_logs_location, customTransactionLogDirectory ) + .newGraphDatabase(); + createTestData( toNodeCount, db ); + db.shutdown(); + + // when + new RestoreDatabaseCommand( fileSystemRule.get(), fromPath, config, databaseName, true ).execute(); + + LogFiles fromStoreLogFiles = LogFilesBuilder.logFilesBasedOnlyBuilder( fromPath, fileSystemRule.get() ).build(); + LogFiles toStoreLogFiles = LogFilesBuilder.logFilesBasedOnlyBuilder( toPath, fileSystemRule.get() ).build(); + LogFiles customLogLocationLogFiles = LogFilesBuilder.logFilesBasedOnlyBuilder( customTxLogDirectory, fileSystemRule.get() ).build(); + assertThat( toStoreLogFiles.logFiles(), emptyArray() ); + assertThat( customLogLocationLogFiles.logFiles(), arrayWithSize( 1 ) ); + assertEquals( fromStoreLogFiles.getLogFileForVersion( 0 ).length(), + customLogLocationLogFiles.getLogFileForVersion( 0 ).length() ); + } + + @Test + public void doNotRemoveRelativeTransactionDirectoryAgain() throws IOException, CommandFailed + { + FileSystemAbstraction fileSystem = Mockito.spy( fileSystemRule.get() ); + File fromPath = directory.directory( "from" ); + File databaseFile = directory.directory(); + File relativeLogDirectory = directory.directory( "relativeDirectory" ); + + Config config = Config.defaults( GraphDatabaseSettings.database_path, databaseFile.getAbsolutePath() ); + config.augment( GraphDatabaseSettings.logical_logs_location, relativeLogDirectory.getAbsolutePath() ); + + createDbAt( fromPath, 10 ); + + new RestoreDatabaseCommand( fileSystem, fromPath, config, "testDatabase", true ).execute(); + + verify( fileSystem ).deleteRecursively( eq( databaseFile ) ); + verify( fileSystem, never() ).deleteRecursively( eq( relativeLogDirectory ) ); + } + @Test public void shouldPrintNiceHelp() throws Throwable { @@ -227,18 +291,24 @@ public void shouldPrintNiceHelp() throws Throwable private static Config configWith( String databaseName, String dataDirectory ) { - return Config.defaults( stringMap( DatabaseManagementSystemSettings.active_database.name(), databaseName, - DatabaseManagementSystemSettings.data_directory.name(), dataDirectory ) ); + return Config.defaults( stringMap( GraphDatabaseSettings.active_database.name(), databaseName, + GraphDatabaseSettings.data_directory.name(), dataDirectory ) ); } private void createDbAt( File fromPath, int nodesToCreate ) { - GraphDatabaseFactory factory = new GraphDatabaseFactory(); - - GraphDatabaseService db = factory.newEmbeddedDatabaseBuilder( fromPath ) - .setConfig( OnlineBackupSettings.online_backup_server, "127.0.0.1:" + PortAuthority.allocatePort() ) + GraphDatabaseService db = new GraphDatabaseFactory().newEmbeddedDatabaseBuilder( fromPath ) + .setConfig( OnlineBackupSettings.online_backup_enabled, Settings.FALSE ) + .setConfig( GraphDatabaseSettings.logical_logs_location, fromPath.getAbsolutePath() ) .newGraphDatabase(); + createTestData( nodesToCreate, db ); + + db.shutdown(); + } + + private void createTestData( int nodesToCreate, GraphDatabaseService db ) + { try ( Transaction tx = db.beginTx() ) { for ( int i = 0; i < nodesToCreate; i++ ) @@ -247,7 +317,5 @@ private void createDbAt( File fromPath, int nodesToCreate ) } tx.success(); } - - db.shutdown(); } } diff --git a/enterprise/causal-clustering/src/main/java/org/neo4j/causalclustering/catchup/storecopy/CopiedStoreRecovery.java b/enterprise/causal-clustering/src/main/java/org/neo4j/causalclustering/catchup/storecopy/CopiedStoreRecovery.java index f78dc5fce4c86..7a2fe16299bf3 100644 --- a/enterprise/causal-clustering/src/main/java/org/neo4j/causalclustering/catchup/storecopy/CopiedStoreRecovery.java +++ b/enterprise/causal-clustering/src/main/java/org/neo4j/causalclustering/catchup/storecopy/CopiedStoreRecovery.java @@ -28,6 +28,7 @@ import org.neo4j.kernel.configuration.Config; import org.neo4j.kernel.configuration.Settings; import org.neo4j.kernel.extension.KernelExtensionFactory; +import org.neo4j.kernel.impl.enterprise.configuration.OnlineBackupSettings; import org.neo4j.kernel.impl.storemigration.UpgradeNotAllowedByConfigurationException; import org.neo4j.kernel.lifecycle.LifecycleAdapter; import org.neo4j.logging.NullLogProvider; @@ -98,8 +99,7 @@ private GraphDatabaseService newTempDatabase( File tempStore ) .setKernelExtensions( kernelExtensions ) .setUserLogProvider( NullLogProvider.getInstance() ) .newEmbeddedDatabaseBuilder( tempStore ) - .setConfig( "dbms.backup.enabled", Settings.FALSE ) - .setConfig( GraphDatabaseSettings.logs_directory, tempStore.getAbsolutePath() ) + .setConfig( OnlineBackupSettings.online_backup_enabled, Settings.FALSE ) .setConfig( GraphDatabaseSettings.keep_logical_logs, Settings.TRUE ) .setConfig( GraphDatabaseSettings.allow_upgrade, config.get( GraphDatabaseSettings.allow_upgrade ).toString() ) diff --git a/enterprise/causal-clustering/src/main/java/org/neo4j/causalclustering/catchup/storecopy/LocalDatabase.java b/enterprise/causal-clustering/src/main/java/org/neo4j/causalclustering/catchup/storecopy/LocalDatabase.java index fb8ff07e2c241..7dbf18370ee48 100644 --- a/enterprise/causal-clustering/src/main/java/org/neo4j/causalclustering/catchup/storecopy/LocalDatabase.java +++ b/enterprise/causal-clustering/src/main/java/org/neo4j/causalclustering/catchup/storecopy/LocalDatabase.java @@ -36,6 +36,7 @@ import org.neo4j.kernel.impl.store.StoreFile; import org.neo4j.kernel.impl.store.StoreType; import org.neo4j.kernel.impl.transaction.log.TransactionAppender; +import org.neo4j.kernel.impl.transaction.log.files.LogFiles; import org.neo4j.kernel.impl.transaction.state.DataSourceManager; import org.neo4j.kernel.impl.util.watcher.FileSystemWatcherService; import org.neo4j.kernel.internal.DatabaseHealth; @@ -67,20 +68,25 @@ public class LocalDatabase implements Lifecycle private volatile AvailabilityRequirement currentRequirement; private volatile TransactionCommitProcess localCommit; - - public LocalDatabase( File storeDir, StoreFiles storeFiles, DataSourceManager dataSourceManager, - Supplier databaseHealthSupplier, FileSystemWatcherService watcherService, + private LogFiles logFiles; + + public LocalDatabase( File storeDir, + StoreFiles storeFiles, + LogFiles logFiles, + DataSourceManager dataSourceManager, + Supplier databaseHealthSupplier, + FileSystemWatcherService watcherService, AvailabilityGuard availabilityGuard, LogProvider logProvider ) { this.storeDir = storeDir; this.storeFiles = storeFiles; + this.logFiles = logFiles; this.dataSourceManager = dataSourceManager; this.databaseHealthSupplier = databaseHealthSupplier; this.availabilityGuard = availabilityGuard; this.watcherService = watcherService; this.log = logProvider.getLog( getClass() ); - raiseAvailabilityGuard( NOT_STOPPED ); } @@ -182,7 +188,7 @@ private DatabaseHealth getDatabaseHealth() public void delete() throws IOException { - storeFiles.delete( storeDir ); + storeFiles.delete( storeDir, logFiles ); } public boolean isEmpty() throws IOException @@ -203,8 +209,8 @@ public File storeDir() void replaceWith( File sourceDir ) throws IOException { - storeFiles.delete( storeDir ); - storeFiles.moveTo( sourceDir, storeDir ); + storeFiles.delete( storeDir, logFiles ); + storeFiles.moveTo( sourceDir, storeDir, logFiles ); } public NeoStoreDataSource dataSource() diff --git a/enterprise/causal-clustering/src/main/java/org/neo4j/causalclustering/catchup/storecopy/RemoteStore.java b/enterprise/causal-clustering/src/main/java/org/neo4j/causalclustering/catchup/storecopy/RemoteStore.java index dddc257dd316c..d754ad7f34647 100644 --- a/enterprise/causal-clustering/src/main/java/org/neo4j/causalclustering/catchup/storecopy/RemoteStore.java +++ b/enterprise/causal-clustering/src/main/java/org/neo4j/causalclustering/catchup/storecopy/RemoteStore.java @@ -32,6 +32,7 @@ import org.neo4j.helpers.AdvertisedSocketAddress; import org.neo4j.io.fs.FileSystemAbstraction; import org.neo4j.io.pagecache.PageCache; +import org.neo4j.kernel.configuration.Config; import org.neo4j.kernel.impl.transaction.CommittedTransactionRepresentation; import org.neo4j.kernel.impl.transaction.log.NoSuchTransactionException; import org.neo4j.kernel.impl.transaction.log.ReadOnlyTransactionIdStore; @@ -53,6 +54,7 @@ public class RemoteStore { private final Log log; + private final Config config; private final Monitors monitors; private final FileSystemAbstraction fs; private final PageCache pageCache; @@ -61,8 +63,8 @@ public class RemoteStore private final TxPullClient txPullClient; private final TransactionLogCatchUpFactory transactionLogFactory; - public RemoteStore( LogProvider logProvider, FileSystemAbstraction fs, PageCache pageCache, StoreCopyClient storeCopyClient, TxPullClient txPullClient, - TransactionLogCatchUpFactory transactionLogFactory, Monitors monitors ) + public RemoteStore( LogProvider logProvider, FileSystemAbstraction fs, PageCache pageCache, StoreCopyClient storeCopyClient, + TxPullClient txPullClient, TransactionLogCatchUpFactory transactionLogFactory, Config config, Monitors monitors ) { this.logProvider = logProvider; this.storeCopyClient = storeCopyClient; @@ -70,6 +72,7 @@ public RemoteStore( LogProvider logProvider, FileSystemAbstraction fs, PageCache this.fs = fs; this.pageCache = pageCache; this.transactionLogFactory = transactionLogFactory; + this.config = config; this.monitors = monitors; this.log = logProvider.getLog( getClass() ); } @@ -97,7 +100,8 @@ private long getPullIndex( File storeDir ) throws IOException log.info( "Last Clean Tx Id: %d", lastCleanTxId ); /* these are the transaction logs */ - ReadOnlyTransactionStore txStore = new ReadOnlyTransactionStore( pageCache, fs, storeDir, new Monitors() ); + ReadOnlyTransactionStore txStore = new ReadOnlyTransactionStore( pageCache, fs, storeDir, config, + new Monitors() ); long lastTxId = BASE_TX_ID; try ( Lifespan ignored = new Lifespan( txStore ); TransactionCursor cursor = txStore.getTransactions( lastCleanTxId ) ) @@ -123,10 +127,11 @@ private long getPullIndex( File storeDir ) throws IOException } } - public CatchupResult tryCatchingUp( AdvertisedSocketAddress from, StoreId expectedStoreId, File storeDir ) throws StoreCopyFailedException, IOException + public CatchupResult tryCatchingUp( AdvertisedSocketAddress from, StoreId expectedStoreId, File storeDir, + boolean keepTxLogsInStoreDir ) throws StoreCopyFailedException, IOException { long pullIndex = getPullIndex( storeDir ); - return pullTransactions( from, expectedStoreId, storeDir, pullIndex, false ); + return pullTransactions( from, expectedStoreId, storeDir, pullIndex, false, keepTxLogsInStoreDir ); } public void copy( AdvertisedSocketAddress from, StoreId expectedStoreId, File destDir ) @@ -143,7 +148,8 @@ public void copy( AdvertisedSocketAddress from, StoreId expectedStoreId, File de log.info( "Store files need to be recovered starting from: %d", lastFlushedTxId ); - CatchupResult catchupResult = pullTransactions( from, expectedStoreId, destDir, lastFlushedTxId, true ); + CatchupResult catchupResult = + pullTransactions( from, expectedStoreId, destDir, lastFlushedTxId, true, true ); if ( catchupResult != SUCCESS_END_OF_STREAM ) { throw new StreamingTransactionsFailedException( "Failed to pull transactions: " + catchupResult ); @@ -155,10 +161,12 @@ public void copy( AdvertisedSocketAddress from, StoreId expectedStoreId, File de } } - private CatchupResult pullTransactions( AdvertisedSocketAddress from, StoreId expectedStoreId, File storeDir, long fromTxId, boolean asPartOfStoreCopy ) + private CatchupResult pullTransactions( AdvertisedSocketAddress from, StoreId expectedStoreId, File storeDir, long fromTxId, + boolean asPartOfStoreCopy, boolean keepTxLogsInStoreDir ) throws IOException, StoreCopyFailedException { - try ( TransactionLogCatchUpWriter writer = transactionLogFactory.create( storeDir, fs, pageCache, logProvider, fromTxId, asPartOfStoreCopy ) ) + try ( TransactionLogCatchUpWriter writer = transactionLogFactory.create( storeDir, fs, pageCache, config, + logProvider, fromTxId, asPartOfStoreCopy, keepTxLogsInStoreDir ) ) { log.info( "Pulling transactions from: %d", fromTxId ); diff --git a/enterprise/causal-clustering/src/main/java/org/neo4j/causalclustering/catchup/storecopy/StoreCopyProcess.java b/enterprise/causal-clustering/src/main/java/org/neo4j/causalclustering/catchup/storecopy/StoreCopyProcess.java index 55f4378ee3698..813fc59d15286 100644 --- a/enterprise/causal-clustering/src/main/java/org/neo4j/causalclustering/catchup/storecopy/StoreCopyProcess.java +++ b/enterprise/causal-clustering/src/main/java/org/neo4j/causalclustering/catchup/storecopy/StoreCopyProcess.java @@ -37,8 +37,8 @@ public class StoreCopyProcess private final Log log; private final RemoteStore remoteStore; - public StoreCopyProcess( FileSystemAbstraction fs, PageCache pageCache, LocalDatabase localDatabase, CopiedStoreRecovery copiedStoreRecovery, - RemoteStore remoteStore, LogProvider logProvider ) + public StoreCopyProcess( FileSystemAbstraction fs, PageCache pageCache, LocalDatabase localDatabase, + CopiedStoreRecovery copiedStoreRecovery, RemoteStore remoteStore, LogProvider logProvider ) { this.fs = fs; this.pageCache = pageCache; diff --git a/enterprise/causal-clustering/src/main/java/org/neo4j/causalclustering/catchup/storecopy/StoreFiles.java b/enterprise/causal-clustering/src/main/java/org/neo4j/causalclustering/catchup/storecopy/StoreFiles.java index 6e7d2dea2c347..09b68f1f76b75 100644 --- a/enterprise/causal-clustering/src/main/java/org/neo4j/causalclustering/catchup/storecopy/StoreFiles.java +++ b/enterprise/causal-clustering/src/main/java/org/neo4j/causalclustering/catchup/storecopy/StoreFiles.java @@ -33,6 +33,7 @@ import org.neo4j.io.fs.FileSystemAbstraction; import org.neo4j.io.pagecache.PageCache; import org.neo4j.kernel.impl.store.MetaDataStore; +import org.neo4j.kernel.impl.transaction.log.files.LogFiles; public class StoreFiles { @@ -60,7 +61,7 @@ public StoreFiles( FileSystemAbstraction fs, PageCache pageCache, FilenameFilter this.fileFilter = fileFilter; } - public void delete( File storeDir ) throws IOException + public void delete( File storeDir, LogFiles logFiles ) throws IOException { // 'files' can be null if the directory doesn't exist. This is fine, we just ignore it then. File[] files = fs.listFiles( storeDir, fileFilter ); @@ -72,6 +73,15 @@ public void delete( File storeDir ) throws IOException } } + File[] txLogs = fs.listFiles( logFiles.logFilesDirectory() ); + if ( txLogs != null ) + { + for ( File txLog : txLogs ) + { + fs.deleteFile( txLog ); + } + } + Iterable iterator = acceptedPageCachedFiles( storeDir )::iterator; for ( FileHandle fh : iterator ) { @@ -94,11 +104,13 @@ private Stream acceptedPageCachedFiles( File storeDir ) throws IOExc } } - public void moveTo( File source, File target ) throws IOException + public void moveTo( File source, File target, LogFiles logFiles ) throws IOException { + fs.mkdirs( logFiles.logFilesDirectory() ); for ( File candidate : fs.listFiles( source, fileFilter ) ) { - fs.moveToDirectory( candidate, target ); + File destination = logFiles.isLogFile( candidate) ? logFiles.logFilesDirectory() : target; + fs.moveToDirectory( candidate, destination ); } Iterable fileHandles = acceptedPageCachedFiles( source )::iterator; diff --git a/enterprise/causal-clustering/src/main/java/org/neo4j/causalclustering/catchup/storecopy/TemporaryStoreDirectory.java b/enterprise/causal-clustering/src/main/java/org/neo4j/causalclustering/catchup/storecopy/TemporaryStoreDirectory.java index ffa8d7cc35ba6..7405c90399caa 100644 --- a/enterprise/causal-clustering/src/main/java/org/neo4j/causalclustering/catchup/storecopy/TemporaryStoreDirectory.java +++ b/enterprise/causal-clustering/src/main/java/org/neo4j/causalclustering/catchup/storecopy/TemporaryStoreDirectory.java @@ -24,6 +24,8 @@ import org.neo4j.io.fs.FileSystemAbstraction; import org.neo4j.io.pagecache.PageCache; +import org.neo4j.kernel.impl.transaction.log.files.LogFiles; +import org.neo4j.kernel.impl.transaction.log.files.LogFilesBuilder; public class TemporaryStoreDirectory implements AutoCloseable { @@ -31,12 +33,14 @@ public class TemporaryStoreDirectory implements AutoCloseable private final File tempStoreDir; private final StoreFiles storeFiles; + private LogFiles tempLogFiles; public TemporaryStoreDirectory( FileSystemAbstraction fs, PageCache pageCache, File parent ) throws IOException { this.tempStoreDir = new File( parent, TEMP_COPY_DIRECTORY_NAME ); storeFiles = new StoreFiles( fs, pageCache, ( directory, name ) -> true ); - storeFiles.delete( tempStoreDir ); + tempLogFiles = LogFilesBuilder.logFilesBasedOnlyBuilder( tempStoreDir, fs ).build(); + storeFiles.delete( tempStoreDir, tempLogFiles ); } public File storeDir() @@ -47,6 +51,6 @@ public File storeDir() @Override public void close() throws IOException { - storeFiles.delete( tempStoreDir ); + storeFiles.delete( tempStoreDir, tempLogFiles ); } } diff --git a/enterprise/causal-clustering/src/main/java/org/neo4j/causalclustering/catchup/tx/TransactionLogCatchUpFactory.java b/enterprise/causal-clustering/src/main/java/org/neo4j/causalclustering/catchup/tx/TransactionLogCatchUpFactory.java index a1ac956bf76ce..818e1980e6a13 100644 --- a/enterprise/causal-clustering/src/main/java/org/neo4j/causalclustering/catchup/tx/TransactionLogCatchUpFactory.java +++ b/enterprise/causal-clustering/src/main/java/org/neo4j/causalclustering/catchup/tx/TransactionLogCatchUpFactory.java @@ -24,13 +24,16 @@ import org.neo4j.io.fs.FileSystemAbstraction; import org.neo4j.io.pagecache.PageCache; +import org.neo4j.kernel.configuration.Config; import org.neo4j.logging.LogProvider; public class TransactionLogCatchUpFactory { public TransactionLogCatchUpWriter create( File storeDir, FileSystemAbstraction fs, PageCache pageCache, - LogProvider logProvider, long fromTxId, boolean asPartOfStoreCopy ) throws IOException + Config config, LogProvider logProvider, long fromTxId, boolean asPartOfStoreCopy, boolean keepTxLogsInStoreDir ) + throws IOException { - return new TransactionLogCatchUpWriter( storeDir, fs, pageCache, logProvider, fromTxId, asPartOfStoreCopy ); + return new TransactionLogCatchUpWriter( storeDir, fs, pageCache, config, logProvider, fromTxId, + asPartOfStoreCopy, keepTxLogsInStoreDir ); } } diff --git a/enterprise/causal-clustering/src/main/java/org/neo4j/causalclustering/catchup/tx/TransactionLogCatchUpWriter.java b/enterprise/causal-clustering/src/main/java/org/neo4j/causalclustering/catchup/tx/TransactionLogCatchUpWriter.java index 4cbc92af6608b..619da11108e43 100644 --- a/enterprise/causal-clustering/src/main/java/org/neo4j/causalclustering/catchup/tx/TransactionLogCatchUpWriter.java +++ b/enterprise/causal-clustering/src/main/java/org/neo4j/causalclustering/catchup/tx/TransactionLogCatchUpWriter.java @@ -24,6 +24,7 @@ import org.neo4j.io.fs.FileSystemAbstraction; import org.neo4j.io.pagecache.PageCache; +import org.neo4j.kernel.configuration.Config; import org.neo4j.kernel.impl.store.MetaDataStore; import org.neo4j.kernel.impl.transaction.CommittedTransactionRepresentation; import org.neo4j.kernel.impl.transaction.log.LogPosition; @@ -52,14 +53,19 @@ public class TransactionLogCatchUpWriter implements TxPullResponseListener, Auto private long lastTxId = -1; private long expectedTxId; - TransactionLogCatchUpWriter( File storeDir, FileSystemAbstraction fs, PageCache pageCache, - LogProvider logProvider, long fromTxId, boolean asPartOfStoreCopy ) throws IOException + TransactionLogCatchUpWriter( File storeDir, FileSystemAbstraction fs, PageCache pageCache, Config config, + LogProvider logProvider, long fromTxId, boolean asPartOfStoreCopy, boolean keepTxLogsInStoreDir ) throws IOException { this.pageCache = pageCache; this.log = logProvider.getLog( getClass() ); this.asPartOfStoreCopy = asPartOfStoreCopy; - this.logFiles = LogFilesBuilder.activeFilesBuilder( storeDir, fs, pageCache ) - .withLastCommittedTransactionIdSupplier( () -> fromTxId - 1 ).build(); + LogFilesBuilder logFilesBuilder = LogFilesBuilder.activeFilesBuilder( storeDir, fs, pageCache ) + .withLastCommittedTransactionIdSupplier( () -> fromTxId - 1 ); + if ( !keepTxLogsInStoreDir ) + { + logFilesBuilder.withConfig( config ); + } + this.logFiles = logFilesBuilder.build(); this.lifespan.add( logFiles ); this.writer = new TransactionLogWriter( new LogEntryWriter( logFiles.getLogFile().getWriter() ) ); this.storeDir = storeDir; diff --git a/enterprise/causal-clustering/src/main/java/org/neo4j/causalclustering/core/EnterpriseCoreEditionModule.java b/enterprise/causal-clustering/src/main/java/org/neo4j/causalclustering/core/EnterpriseCoreEditionModule.java index 10ad2912ba150..d58cdcf27bbcc 100644 --- a/enterprise/causal-clustering/src/main/java/org/neo4j/causalclustering/core/EnterpriseCoreEditionModule.java +++ b/enterprise/causal-clustering/src/main/java/org/neo4j/causalclustering/core/EnterpriseCoreEditionModule.java @@ -22,6 +22,7 @@ import java.io.File; import java.io.FileNotFoundException; import java.io.FileOutputStream; +import java.io.IOException; import java.io.PrintWriter; import java.util.function.Predicate; import java.util.function.Supplier; @@ -63,7 +64,6 @@ import org.neo4j.causalclustering.messaging.RaftOutbound; import org.neo4j.causalclustering.messaging.SenderService; import org.neo4j.com.storecopy.StoreUtil; -import org.neo4j.dbms.DatabaseManagementSystemSettings; import org.neo4j.function.Predicates; import org.neo4j.graphdb.DependencyResolver; import org.neo4j.graphdb.factory.GraphDatabaseSettings; @@ -92,6 +92,8 @@ import org.neo4j.kernel.impl.store.id.IdGeneratorFactory; import org.neo4j.kernel.impl.store.id.IdReuseEligibility; import org.neo4j.kernel.impl.transaction.TransactionHeaderInformationFactory; +import org.neo4j.kernel.impl.transaction.log.files.LogFiles; +import org.neo4j.kernel.impl.transaction.log.files.LogFilesBuilder; import org.neo4j.kernel.impl.transaction.log.files.TransactionLogFiles; import org.neo4j.kernel.impl.util.Dependencies; import org.neo4j.kernel.internal.DatabaseHealth; @@ -102,7 +104,6 @@ import org.neo4j.kernel.lifecycle.LifecycleStatus; import org.neo4j.kernel.monitoring.Monitors; import org.neo4j.logging.LogProvider; -import org.neo4j.ssl.SslPolicy; import org.neo4j.time.Clocks; import org.neo4j.udc.UsageData; @@ -172,7 +173,7 @@ public void registerEditionSpecificProcedures( Procedures procedures ) throws Ke final LifeSupport life = platformModule.life; final Monitors monitors = platformModule.monitors; - final File dataDir = config.get( DatabaseManagementSystemSettings.data_directory ); + final File dataDir = config.get( GraphDatabaseSettings.data_directory ); final ClusterStateDirectory clusterStateDirectory = new ClusterStateDirectory( dataDir, storeDir, false ); try { @@ -192,8 +193,10 @@ public void registerEditionSpecificProcedures( Procedures procedures ) throws Ke watcherService = createFileSystemWatcherService( fileSystem, storeDir, logging, platformModule.jobScheduler, fileWatcherFileNameFilter() ); dependencies.satisfyDependencies( watcherService ); + LogFiles logFiles = buildLocalDatabaseLogFiles( platformModule, fileSystem, storeDir ); LocalDatabase localDatabase = new LocalDatabase( platformModule.storeDir, new StoreFiles( fileSystem, platformModule.pageCache ), + logFiles, platformModule.dataSourceManager, databaseHealthSupplier, watcherService, @@ -264,6 +267,19 @@ public void registerEditionSpecificProcedures( Procedures procedures ) throws Ke life.add( coreServerModule.membershipWaiterLifecycle ); } + private LogFiles buildLocalDatabaseLogFiles( PlatformModule platformModule, FileSystemAbstraction fileSystem, + File storeDir ) + { + try + { + return LogFilesBuilder.activeFilesBuilder( storeDir, fileSystem, platformModule.pageCache ).withConfig( config ).build(); + } + catch ( IOException e ) + { + throw new RuntimeException( e ); + } + } + protected ClusteringModule getClusteringModule( PlatformModule platformModule, DiscoveryServiceFactory discoveryServiceFactory, ClusterStateDirectory clusterStateDirectory, diff --git a/enterprise/causal-clustering/src/main/java/org/neo4j/causalclustering/core/server/CoreServerModule.java b/enterprise/causal-clustering/src/main/java/org/neo4j/causalclustering/core/server/CoreServerModule.java index 70de812ddf123..fa296c20ec64c 100644 --- a/enterprise/causal-clustering/src/main/java/org/neo4j/causalclustering/core/server/CoreServerModule.java +++ b/enterprise/causal-clustering/src/main/java/org/neo4j/causalclustering/core/server/CoreServerModule.java @@ -126,13 +126,13 @@ public CoreServerModule( IdentityModule identityModule, final PlatformModule pla RemoteStore remoteStore = new RemoteStore( logProvider, fileSystem, platformModule.pageCache, new StoreCopyClient( catchUpClient, logProvider ), new TxPullClient( catchUpClient, platformModule.monitors ), new TransactionLogCatchUpFactory(), - platformModule.monitors ); + config, platformModule.monitors ); CopiedStoreRecovery copiedStoreRecovery = new CopiedStoreRecovery( config, platformModule.kernelExtensions.listFactories(), platformModule.pageCache ); life.add( copiedStoreRecovery ); - StoreCopyProcess storeCopyProcess = - new StoreCopyProcess( fileSystem, platformModule.pageCache, localDatabase, copiedStoreRecovery, remoteStore, logProvider ); + StoreCopyProcess storeCopyProcess = new StoreCopyProcess( fileSystem, platformModule.pageCache, localDatabase, + copiedStoreRecovery, remoteStore, logProvider ); LifeSupport servicesToStopOnStoreCopy = new LifeSupport(); diff --git a/enterprise/causal-clustering/src/main/java/org/neo4j/causalclustering/core/state/CoreBootstrapper.java b/enterprise/causal-clustering/src/main/java/org/neo4j/causalclustering/core/state/CoreBootstrapper.java index 8f9a5a963353f..002f25736b713 100644 --- a/enterprise/causal-clustering/src/main/java/org/neo4j/causalclustering/core/state/CoreBootstrapper.java +++ b/enterprise/causal-clustering/src/main/java/org/neo4j/causalclustering/core/state/CoreBootstrapper.java @@ -125,6 +125,7 @@ private void appendNullTransactionLogEntryToSetRaftIndexToMinusOne() throws IOEx { ReadOnlyTransactionIdStore readOnlyTransactionIdStore = new ReadOnlyTransactionIdStore( pageCache, storeDir ); LogFiles logFiles = LogFilesBuilder.activeFilesBuilder( storeDir, fs, pageCache ) + .withConfig( config ) .withLastCommittedTransactionIdSupplier( () -> readOnlyTransactionIdStore.getLastClosedTransactionId() - 1 ) .build(); diff --git a/enterprise/causal-clustering/src/main/java/org/neo4j/causalclustering/core/state/snapshot/CoreStateDownloader.java b/enterprise/causal-clustering/src/main/java/org/neo4j/causalclustering/core/state/snapshot/CoreStateDownloader.java index 54230d982a67d..3675f6f32cb31 100644 --- a/enterprise/causal-clustering/src/main/java/org/neo4j/causalclustering/core/state/snapshot/CoreStateDownloader.java +++ b/enterprise/causal-clustering/src/main/java/org/neo4j/causalclustering/core/state/snapshot/CoreStateDownloader.java @@ -127,7 +127,8 @@ public void onCoreSnapshot( CompletableFuture signal, CoreSnapshot else { StoreId localStoreId = localDatabase.storeId(); - CatchupResult catchupResult = remoteStore.tryCatchingUp( fromAddress, localStoreId, localDatabase.storeDir() ); + CatchupResult catchupResult = remoteStore.tryCatchingUp( fromAddress, localStoreId, localDatabase + .storeDir(), false ); if ( catchupResult == E_TRANSACTION_PRUNED ) { diff --git a/enterprise/causal-clustering/src/main/java/org/neo4j/causalclustering/readreplica/EnterpriseReadReplicaEditionModule.java b/enterprise/causal-clustering/src/main/java/org/neo4j/causalclustering/readreplica/EnterpriseReadReplicaEditionModule.java index 7e1c4e0f74efb..f76fe6f08ea87 100644 --- a/enterprise/causal-clustering/src/main/java/org/neo4j/causalclustering/readreplica/EnterpriseReadReplicaEditionModule.java +++ b/enterprise/causal-clustering/src/main/java/org/neo4j/causalclustering/readreplica/EnterpriseReadReplicaEditionModule.java @@ -20,6 +20,7 @@ package org.neo4j.causalclustering.readreplica; import java.io.File; +import java.io.IOException; import java.util.HashMap; import java.util.Iterator; import java.util.Map; @@ -97,6 +98,8 @@ import org.neo4j.kernel.impl.transaction.log.LogicalTransactionStore; import org.neo4j.kernel.impl.transaction.log.TransactionAppender; import org.neo4j.kernel.impl.transaction.log.TransactionIdStore; +import org.neo4j.kernel.impl.transaction.log.files.LogFiles; +import org.neo4j.kernel.impl.transaction.log.files.LogFilesBuilder; import org.neo4j.kernel.impl.transaction.log.files.TransactionLogFiles; import org.neo4j.kernel.impl.util.Dependencies; import org.neo4j.kernel.internal.DatabaseHealth; @@ -217,14 +220,16 @@ public class EnterpriseReadReplicaEditionModule extends EditionModule DelayedRenewableTimeoutService catchupTimeoutService = new DelayedRenewableTimeoutService( Clocks.systemClock(), logProvider ); StoreFiles storeFiles = new StoreFiles( fileSystem, pageCache ); + LogFiles logFiles = buildLocalDatabaseLogFiles( platformModule, fileSystem, storeDir, config ); LocalDatabase localDatabase = - new LocalDatabase( platformModule.storeDir, storeFiles, platformModule.dataSourceManager, databaseHealthSupplier, watcherService, - platformModule.availabilityGuard, logProvider ); + new LocalDatabase( platformModule.storeDir, storeFiles, logFiles, platformModule.dataSourceManager, + databaseHealthSupplier, + watcherService, platformModule.availabilityGuard, logProvider ); RemoteStore remoteStore = new RemoteStore( platformModule.logging.getInternalLogProvider(), fileSystem, platformModule.pageCache, new StoreCopyClient( catchUpClient, logProvider ), new TxPullClient( catchUpClient, platformModule.monitors ), - new TransactionLogCatchUpFactory(), platformModule.monitors ); + new TransactionLogCatchUpFactory(), config, platformModule.monitors ); CopiedStoreRecovery copiedStoreRecovery = new CopiedStoreRecovery( config, platformModule.kernelExtensions.listFactories(), platformModule.pageCache ); @@ -232,7 +237,8 @@ public class EnterpriseReadReplicaEditionModule extends EditionModule LifeSupport servicesToStopOnStoreCopy = new LifeSupport(); - StoreCopyProcess storeCopyProcess = new StoreCopyProcess( fileSystem, pageCache, localDatabase, copiedStoreRecovery, remoteStore, logProvider ); + StoreCopyProcess storeCopyProcess = new StoreCopyProcess( fileSystem, pageCache, localDatabase, + copiedStoreRecovery, remoteStore, logProvider ); ConnectToRandomCoreServerStrategy defaultStrategy = new ConnectToRandomCoreServerStrategy(); defaultStrategy.inject( topologyService, config, logProvider, myself ); @@ -375,4 +381,17 @@ private static Map backupDisabledSettings() overrideBackupSettings.put( OnlineBackupSettings.online_backup_enabled.name(), Settings.FALSE ); return overrideBackupSettings; } + + private LogFiles buildLocalDatabaseLogFiles( PlatformModule platformModule, FileSystemAbstraction fileSystem, + File storeDir, Config config ) + { + try + { + return LogFilesBuilder.activeFilesBuilder( storeDir, fileSystem, platformModule.pageCache ).withConfig( config ).build(); + } + catch ( IOException e ) + { + throw new RuntimeException( e ); + } + } } diff --git a/enterprise/causal-clustering/src/main/java/org/neo4j/commandline/dbms/UnbindFromClusterCommand.java b/enterprise/causal-clustering/src/main/java/org/neo4j/commandline/dbms/UnbindFromClusterCommand.java index 034a87226ffc0..5dd022531b7fe 100644 --- a/enterprise/causal-clustering/src/main/java/org/neo4j/commandline/dbms/UnbindFromClusterCommand.java +++ b/enterprise/causal-clustering/src/main/java/org/neo4j/commandline/dbms/UnbindFromClusterCommand.java @@ -31,7 +31,7 @@ import org.neo4j.commandline.admin.IncorrectUsage; import org.neo4j.commandline.admin.OutsideWorld; import org.neo4j.commandline.arguments.Arguments; -import org.neo4j.dbms.DatabaseManagementSystemSettings; +import org.neo4j.graphdb.factory.GraphDatabaseSettings; import org.neo4j.io.fs.FileUtils; import org.neo4j.kernel.StoreLockException; import org.neo4j.kernel.configuration.Config; @@ -61,7 +61,7 @@ static Arguments arguments() private static Config loadNeo4jConfig( Path homeDir, Path configDir, String databaseName ) { return fromFile( configDir.resolve( Config.DEFAULT_CONFIG_FILE_NAME ) ) - .withSetting( DatabaseManagementSystemSettings.active_database, databaseName ) + .withSetting( GraphDatabaseSettings.active_database, databaseName ) .withHome( homeDir ).build(); } @@ -71,8 +71,8 @@ public void execute( String[] args ) throws IncorrectUsage, CommandFailed try { Config config = loadNeo4jConfig( homeDir, configDir, arguments.parse( args ).get( "database" ) ); - File dataDirectory = config.get( DatabaseManagementSystemSettings.data_directory ); - Path pathToSpecificDatabase = config.get( DatabaseManagementSystemSettings.database_path ).toPath(); + File dataDirectory = config.get( GraphDatabaseSettings.data_directory ); + Path pathToSpecificDatabase = config.get( GraphDatabaseSettings.database_path ).toPath(); boolean hasDatabase = true; try diff --git a/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/backup/RestoreClusterUtils.java b/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/backup/RestoreClusterUtils.java index 7a801fdf709ac..fd2ed23d76ab2 100644 --- a/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/backup/RestoreClusterUtils.java +++ b/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/backup/RestoreClusterUtils.java @@ -19,6 +19,8 @@ */ package org.neo4j.causalclustering.backup; +import org.apache.commons.lang3.StringUtils; + import java.io.File; import org.neo4j.graphdb.GraphDatabaseService; @@ -37,14 +39,22 @@ private RestoreClusterUtils() { } - public static File createClassicNeo4jStore( File base, FileSystemAbstraction fileSystem, int nodesToCreate, String recordFormat ) + public static File createClassicNeo4jStore( File base, FileSystemAbstraction fileSystem, int nodesToCreate, + String recordFormat ) + { + return createClassicNeo4jStore( base, fileSystem, nodesToCreate, recordFormat, StringUtils.EMPTY ); + } + + public static File createClassicNeo4jStore( File base, FileSystemAbstraction fileSystem, + int nodesToCreate, String recordFormat, String logicalLogsLocation ) { - File existingDbDir = new File( base, "existing" ); + File storeDir = new File( base, "existing" ); GraphDatabaseService db = new TestGraphDatabaseFactory() .setFileSystem( fileSystem ) - .newEmbeddedDatabaseBuilder( existingDbDir ) + .newEmbeddedDatabaseBuilder( storeDir ) .setConfig( GraphDatabaseSettings.record_format, recordFormat ) .setConfig( OnlineBackupSettings.online_backup_enabled, Boolean.FALSE.toString() ) + .setConfig( GraphDatabaseSettings.logical_logs_location, logicalLogsLocation ) .newGraphDatabase(); for ( int i = 0; i < (nodesToCreate / 2); i++ ) @@ -60,6 +70,6 @@ public static File createClassicNeo4jStore( File base, FileSystemAbstraction fil db.shutdown(); - return existingDbDir; + return storeDir; } } diff --git a/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/catchup/storecopy/CopiedStoreRecoveryTest.java b/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/catchup/storecopy/CopiedStoreRecoveryTest.java index ee8d1b5c0929f..39f255c4231f7 100644 --- a/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/catchup/storecopy/CopiedStoreRecoveryTest.java +++ b/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/catchup/storecopy/CopiedStoreRecoveryTest.java @@ -22,6 +22,7 @@ import org.junit.Test; import java.io.File; +import java.io.IOException; import org.neo4j.helpers.collection.Iterables; import org.neo4j.io.pagecache.PageCache; @@ -34,7 +35,7 @@ public class CopiedStoreRecoveryTest { @Test - public void shouldThrowIfAlreadyShutdown() + public void shouldThrowIfAlreadyShutdown() throws IOException { // Given CopiedStoreRecovery copiedStoreRecovery = diff --git a/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/catchup/storecopy/LocalDatabaseTest.java b/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/catchup/storecopy/LocalDatabaseTest.java index 1cb11827738a8..0cf9f55c6f998 100644 --- a/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/catchup/storecopy/LocalDatabaseTest.java +++ b/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/catchup/storecopy/LocalDatabaseTest.java @@ -26,6 +26,7 @@ import java.time.Clock; import org.neo4j.kernel.AvailabilityGuard; +import org.neo4j.kernel.impl.transaction.log.files.LogFiles; import org.neo4j.kernel.impl.transaction.state.DataSourceManager; import org.neo4j.kernel.impl.util.watcher.FileSystemWatcherService; import org.neo4j.kernel.internal.DatabaseHealth; @@ -167,7 +168,7 @@ private static LocalDatabase newLocalDatabase( AvailabilityGuard availabilityGua private static LocalDatabase newLocalDatabase( AvailabilityGuard availabilityGuard, DataSourceManager dataSourceManager, FileSystemWatcherService fileWatcher ) { - return new LocalDatabase( new File( "." ), mock( StoreFiles.class ), dataSourceManager, + return new LocalDatabase( new File( "." ), mock( StoreFiles.class ), mock( LogFiles.class ), dataSourceManager, () -> mock( DatabaseHealth.class ), fileWatcher, availabilityGuard, NullLogProvider.getInstance() ); } diff --git a/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/catchup/storecopy/RemoteStoreTest.java b/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/catchup/storecopy/RemoteStoreTest.java index 758ce49cf4fb7..9cd41137a9f33 100644 --- a/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/catchup/storecopy/RemoteStoreTest.java +++ b/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/catchup/storecopy/RemoteStoreTest.java @@ -31,6 +31,7 @@ import org.neo4j.causalclustering.identity.StoreId; import org.neo4j.helpers.AdvertisedSocketAddress; import org.neo4j.io.fs.FileSystemAbstraction; +import org.neo4j.kernel.configuration.Config; import org.neo4j.kernel.monitoring.Monitors; import org.neo4j.logging.LogProvider; import org.neo4j.logging.NullLogProvider; @@ -60,7 +61,7 @@ public void shouldCopyStoreFilesAndPullTransactions() throws Exception TransactionLogCatchUpWriter writer = mock( TransactionLogCatchUpWriter.class ); RemoteStore remoteStore = new RemoteStore( NullLogProvider.getInstance(), mock( FileSystemAbstraction.class ), - null, storeCopyClient, txPullClient, factory( writer ), new Monitors() ); + null, storeCopyClient, txPullClient, factory( writer ), Config.defaults(), new Monitors() ); // when AdvertisedSocketAddress localhost = new AdvertisedSocketAddress( "127.0.0.1", 1234 ); @@ -90,7 +91,7 @@ public void shouldSetLastPulledTransactionId() throws Exception TransactionLogCatchUpWriter writer = mock( TransactionLogCatchUpWriter.class ); RemoteStore remoteStore = new RemoteStore( NullLogProvider.getInstance(), mock( FileSystemAbstraction.class ), - null, storeCopyClient, txPullClient, factory( writer ), new Monitors() ); + null, storeCopyClient, txPullClient, factory( writer ), Config.defaults(), new Monitors() ); // when remoteStore.copy( localhost, wantedStoreId, new File( "destination" ) ); @@ -112,7 +113,7 @@ public void shouldCloseDownTxLogWriterIfTxStreamingFails() throws Exception RemoteStore remoteStore = new RemoteStore( NullLogProvider.getInstance(), mock( FileSystemAbstraction.class ), null, - storeCopyClient, txPullClient, factory( writer ), new Monitors() ); + storeCopyClient, txPullClient, factory( writer ), Config.defaults(), new Monitors() ); doThrow( StoreCopyFailedException.class ).when( txPullClient ) .pullTransactions( isNull(), eq( storeId ), anyLong(), any() ); @@ -134,8 +135,8 @@ public void shouldCloseDownTxLogWriterIfTxStreamingFails() throws Exception private TransactionLogCatchUpFactory factory( TransactionLogCatchUpWriter writer ) throws IOException { TransactionLogCatchUpFactory factory = mock( TransactionLogCatchUpFactory.class ); - when( factory.create( isNull(), any( FileSystemAbstraction.class ), - isNull(), any( LogProvider.class ), anyLong(), anyBoolean() ) ).thenReturn( writer ); + when( factory.create( isNull(), any( FileSystemAbstraction.class ), isNull(), any( Config.class ), + any( LogProvider.class ), anyLong(), anyBoolean(), anyBoolean() ) ).thenReturn( writer ); return factory; } } diff --git a/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/catchup/storecopy/StoreFilesTest.java b/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/catchup/storecopy/StoreFilesTest.java index f2126298fbf01..f16db942b31a7 100644 --- a/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/catchup/storecopy/StoreFilesTest.java +++ b/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/catchup/storecopy/StoreFilesTest.java @@ -40,6 +40,8 @@ import org.neo4j.io.pagecache.PageCache; import org.neo4j.kernel.impl.store.MetaDataStore; import org.neo4j.kernel.impl.store.MetaDataStore.Position; +import org.neo4j.kernel.impl.transaction.log.files.LogFiles; +import org.neo4j.kernel.impl.transaction.log.files.LogFilesBuilder; import org.neo4j.test.rule.PageCacheRule; import org.neo4j.test.rule.TestDirectory; import org.neo4j.test.rule.fs.EphemeralFileSystemRule; @@ -67,6 +69,7 @@ public class StoreFilesTest private EphemeralFileSystemAbstraction pc; private PageCache pageCache; private StoreFiles storeFiles; + private LogFiles logFiles; public StoreFilesTest() { @@ -93,6 +96,7 @@ public void setUp() throws Exception pc = hiddenFileSystemRule.get(); pageCache = pageCacheRule.getPageCache( pc ); storeFiles = new StoreFiles( fs, pageCache ); + logFiles = LogFilesBuilder.logFilesBasedOnlyBuilder( testDirectory.directory(), fs ).build(); } private void createOnFileSystem( File file ) throws IOException @@ -128,7 +132,7 @@ public void deleteMustRecursivelyRemoveFilesInGivenDirectory() throws Exception assertTrue( fs.fileExists( a ) ); assertTrue( pc.fileExists( b ) ); - storeFiles.delete( dir ); + storeFiles.delete( dir, logFiles ); assertFalse( fs.fileExists( a ) ); assertFalse( pc.fileExists( b ) ); @@ -150,7 +154,7 @@ public void deleteMustNotDeleteIgnoredFiles() throws Exception FilenameFilter filter = ( directory, name ) -> !name.equals( "c" ) && !name.equals( "d" ); storeFiles = new StoreFiles( fs, pageCache, filter ); - storeFiles.delete( dir ); + storeFiles.delete( dir, logFiles ); assertFalse( fs.fileExists( a ) ); assertFalse( pc.fileExists( b ) ); @@ -175,7 +179,7 @@ public void deleteMustNotDeleteFilesInIgnoredDirectories() throws Exception FilenameFilter filter = ( directory, name ) -> !name.startsWith( "ignore" ); storeFiles = new StoreFiles( fs, pageCache, filter ); - storeFiles.delete( dir ); + storeFiles.delete( dir, logFiles ); assertFalse( fs.fileExists( a ) ); assertFalse( pc.fileExists( b ) ); @@ -189,7 +193,7 @@ public void deleteMustSilentlyIgnoreMissingDirectories() throws Exception File dir = getBaseDir(); File sub = new File( dir, "sub" ); - storeFiles.delete( sub ); + storeFiles.delete( sub, logFiles ); } @Test @@ -208,7 +212,7 @@ public void mustMoveFilesToTargetDirectory() throws Exception createOnFileSystem( new File( tgt, ".fs-ignore" ) ); createOnPageCache( new File( tgt, ".pc-ignore" ) ); - storeFiles.moveTo( src, tgt ); + storeFiles.moveTo( src, tgt, logFiles ); assertFalse( fs.fileExists( a ) ); assertFalse( pc.fileExists( b ) ); @@ -233,7 +237,7 @@ public void movedFilesMustRetainTheirRelativePaths() throws Exception createOnFileSystem( new File( tgt, ".fs-ignore" ) ); createOnPageCache( new File( tgt, ".pc-ignore" ) ); - storeFiles.moveTo( src, tgt ); + storeFiles.moveTo( src, tgt, logFiles ); assertFalse( fs.fileExists( a ) ); assertFalse( pc.fileExists( b ) ); @@ -264,7 +268,7 @@ public void moveMustIgnoreFilesFilteredOut() throws Exception FilenameFilter filter = ( directory, name ) -> !name.startsWith( "ignore" ); storeFiles = new StoreFiles( fs, pageCache, filter ); - storeFiles.moveTo( src, tgt ); + storeFiles.moveTo( src, tgt, logFiles ); assertFalse( fs.fileExists( a ) ); assertFalse( pc.fileExists( b ) ); diff --git a/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/catchup/tx/TransactionLogCatchUpWriterTest.java b/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/catchup/tx/TransactionLogCatchUpWriterTest.java index 332e54cad6a93..28569a0044844 100644 --- a/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/catchup/tx/TransactionLogCatchUpWriterTest.java +++ b/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/catchup/tx/TransactionLogCatchUpWriterTest.java @@ -22,14 +22,20 @@ import org.junit.Before; import org.junit.Rule; import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; import java.io.File; import java.io.IOException; +import java.util.Arrays; +import java.util.List; import org.neo4j.causalclustering.identity.StoreId; +import org.neo4j.graphdb.factory.GraphDatabaseSettings; import org.neo4j.io.fs.FileSystemAbstraction; import org.neo4j.io.pagecache.PageCache; import org.neo4j.kernel.NeoStoreDataSource; +import org.neo4j.kernel.configuration.Config; import org.neo4j.kernel.impl.storageengine.impl.recordstorage.RecordStorageCommandReaderFactory; import org.neo4j.kernel.impl.transaction.CommittedTransactionRepresentation; import org.neo4j.kernel.impl.transaction.command.Commands; @@ -64,24 +70,31 @@ import static org.junit.Assert.assertTrue; import static org.neo4j.kernel.impl.transaction.command.Commands.createNode; +@RunWith( Parameterized.class ) public class TransactionLogCatchUpWriterTest { @Rule - public final TestDirectory dir = TestDirectory.testDirectory( getClass() ); - + public final TestDirectory dir = TestDirectory.testDirectory(); @Rule public final DefaultFileSystemRule fsRule = new DefaultFileSystemRule(); - @Rule public final PageCacheRule pageCacheRule = new PageCacheRule(); - @Rule public NeoStoreDataSourceRule dsRule = new NeoStoreDataSourceRule(); + @Parameterized.Parameter + public boolean partOfStoreCopy; + private PageCache pageCache; private FileSystemAbstraction fs; private File storeDir; + @Parameterized.Parameters + public static List partOfStoreCopy() + { + return Arrays.asList( Boolean.TRUE, Boolean.FALSE ); + } + @Before public void setup() throws IOException { @@ -93,14 +106,25 @@ public void setup() throws IOException @Test public void shouldCreateTransactionLogWithCheckpoint() throws Exception { - // given + createTransactionLogWithCheckpoint( Config.defaults(), true ); + } + + @Test + public void createTransactionLogWithCheckpointInCustomLocation() throws IOException + { + createTransactionLogWithCheckpoint( Config.defaults( GraphDatabaseSettings.logical_logs_location, + "custom-tx-logs"), false ); + } + + private void createTransactionLogWithCheckpoint( Config config, boolean logsInStoreDir ) throws IOException + { org.neo4j.kernel.impl.store.StoreId storeId = simulateStoreCopy(); int fromTxId = 37; int endTxId = fromTxId + 5; - TransactionLogCatchUpWriter catchUpWriter = new TransactionLogCatchUpWriter( storeDir, fs, pageCache, - NullLogProvider.getInstance(), fromTxId, true ); + TransactionLogCatchUpWriter catchUpWriter = new TransactionLogCatchUpWriter( storeDir, fs, pageCache, config, + NullLogProvider.getInstance(), fromTxId, partOfStoreCopy, logsInStoreDir ); // when for ( int i = fromTxId; i <= endTxId; i++ ) @@ -111,15 +135,24 @@ public void shouldCreateTransactionLogWithCheckpoint() throws Exception catchUpWriter.close(); // then - verifyTransactionsInLog( fromTxId, endTxId ); - verifyCheckpointInLog(); // necessary for recovery + LogFilesBuilder logFilesBuilder = LogFilesBuilder.activeFilesBuilder( storeDir, fs, pageCache ); + if ( !logsInStoreDir ) + { + logFilesBuilder.withConfig( config ); + } + LogFiles logFiles = logFilesBuilder.build(); + + verifyTransactionsInLog( logFiles, fromTxId, endTxId ); + if ( partOfStoreCopy ) + { + verifyCheckpointInLog( logFiles ); + } } - private void verifyCheckpointInLog() throws IOException + private void verifyCheckpointInLog( LogFiles logFiles ) { LogEntryReader logEntryReader = new VersionAwareLogEntryReader<>( new RecordStorageCommandReaderFactory(), InvalidLogEntryHandler.STRICT ); - LogFiles logFiles = LogFilesBuilder.activeFilesBuilder( storeDir, fs, pageCache ).withLogEntryReader( logEntryReader ).build(); final LogTailScanner logTailScanner = new LogTailScanner( logFiles, logEntryReader, new Monitors() ); LogTailInformation tailInformation = logTailScanner.getTailInformation(); @@ -127,10 +160,10 @@ private void verifyCheckpointInLog() throws IOException assertTrue( tailInformation.commitsAfterLastCheckpoint() ); } - private void verifyTransactionsInLog( long fromTxId, long endTxId ) throws IOException + private void verifyTransactionsInLog( LogFiles logFiles, long fromTxId, long endTxId ) throws + IOException { long expectedTxId = fromTxId; - LogFiles logFiles = LogFilesBuilder.activeFilesBuilder( storeDir, fs, pageCache ).build(); LogVersionedStoreChannel versionedStoreChannel = logFiles.openForVersion( 0 ); try ( ReadableLogChannel channel = new ReadAheadLogChannel( versionedStoreChannel, LogVersionBridge.NO_MORE_CHANNELS, 1024 ) ) diff --git a/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/core/state/CoreBootstrapperIT.java b/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/core/state/CoreBootstrapperIT.java index 9873f5e597ab2..e93188dc449d8 100644 --- a/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/core/state/CoreBootstrapperIT.java +++ b/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/core/state/CoreBootstrapperIT.java @@ -24,6 +24,7 @@ import org.junit.rules.RuleChain; import java.io.File; +import java.io.IOException; import java.util.Set; import org.neo4j.causalclustering.backup.RestoreClusterUtils; @@ -35,6 +36,7 @@ import org.neo4j.causalclustering.core.state.snapshot.CoreStateType; import org.neo4j.causalclustering.core.state.snapshot.RaftCoreState; import org.neo4j.causalclustering.identity.MemberId; +import org.neo4j.graphdb.factory.GraphDatabaseSettings; import org.neo4j.io.fs.FileSystemAbstraction; import org.neo4j.io.pagecache.PageCache; import org.neo4j.kernel.configuration.Config; @@ -48,15 +50,13 @@ import org.neo4j.test.rule.TestDirectory; import org.neo4j.test.rule.fs.DefaultFileSystemRule; +import static java.lang.Integer.parseInt; +import static java.util.UUID.randomUUID; import static org.hamcrest.Matchers.allOf; import static org.hamcrest.Matchers.greaterThanOrEqualTo; import static org.hamcrest.Matchers.lessThanOrEqualTo; - -import static java.lang.Integer.parseInt; -import static java.util.UUID.randomUUID; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertThat; - import static org.neo4j.graphdb.factory.GraphDatabaseSettings.record_id_batch_size; import static org.neo4j.helpers.collection.Iterators.asSet; @@ -82,7 +82,32 @@ public void shouldSetAllCoreState() throws Exception PageCache pageCache = pageCacheRule.getPageCache( fileSystem ); CoreBootstrapper bootstrapper = new CoreBootstrapper( classicNeo4jStore, pageCache, fileSystem, Config.defaults(), NullLogProvider.getInstance() ); + bootstrapAndVerify( nodeCount, fileSystem, classicNeo4jStore, pageCache, Config.defaults(), bootstrapper ); + } + @Test + public void setAllCoreStateOnDatabaseWithCustomLogFilesLocation() throws Exception + { + // given + int nodeCount = 100; + FileSystemAbstraction fileSystem = fileSystemRule.get(); + File baseDirectory = testDirectory.directory(); + String customTransactionLogsLocation = "transaction-logs"; + File classicNeo4jStore = RestoreClusterUtils.createClassicNeo4jStore( baseDirectory, fileSystem, nodeCount, + Standard.LATEST_NAME, customTransactionLogsLocation ); + + PageCache pageCache = pageCacheRule.getPageCache( fileSystem ); + Config config = Config.defaults( GraphDatabaseSettings.logical_logs_location, + customTransactionLogsLocation ); + CoreBootstrapper bootstrapper = new CoreBootstrapper( classicNeo4jStore, pageCache, fileSystem, + config, NullLogProvider.getInstance() ); + + bootstrapAndVerify( nodeCount, fileSystem, classicNeo4jStore, pageCache, config, bootstrapper ); + } + + private void bootstrapAndVerify( long nodeCount, FileSystemAbstraction fileSystem, File classicNeo4jStore, + PageCache pageCache, Config config, CoreBootstrapper bootstrapper ) throws IOException + { // when Set membership = asSet( randomMember(), randomMember(), randomMember() ); CoreSnapshot snapshot = bootstrapper.bootstrap( membership ); @@ -90,7 +115,7 @@ public void shouldSetAllCoreState() throws Exception // then int recordIdBatchSize = parseInt( record_id_batch_size.getDefaultValue() ); assertThat( ((IdAllocationState) snapshot.get( CoreStateType.ID_ALLOCATION )).firstUnallocated( IdType.NODE ), - allOf( greaterThanOrEqualTo( (long) nodeCount ), lessThanOrEqualTo( (long) nodeCount + recordIdBatchSize ) ) ); + allOf( greaterThanOrEqualTo( nodeCount ), lessThanOrEqualTo( nodeCount + recordIdBatchSize ) ) ); /* Bootstrapped state is created in RAFT land at index -1 and term -1. */ assertEquals( 0, snapshot.prevIndex() ); @@ -105,10 +130,11 @@ public void shouldSetAllCoreState() throws Exception /* The session state is initially empty. */ assertEquals( new GlobalSessionTrackerState(), snapshot.get( CoreStateType.SESSION_TRACKER ) ); + ReadOnlyTransactionStore transactionStore = new ReadOnlyTransactionStore( pageCache, fileSystem, + classicNeo4jStore, config, new Monitors() ); LastCommittedIndexFinder lastCommittedIndexFinder = new LastCommittedIndexFinder( new ReadOnlyTransactionIdStore( pageCache, classicNeo4jStore ), - new ReadOnlyTransactionStore( pageCache, fileSystem, classicNeo4jStore, new Monitors() ), - NullLogProvider.getInstance() ); + transactionStore, NullLogProvider.getInstance() ); long lastCommittedIndex = lastCommittedIndexFinder.getLastCommittedIndex(); assertEquals( -1, lastCommittedIndex ); diff --git a/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/core/state/machines/id/RebuildReplicatedIdGeneratorsTest.java b/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/core/state/machines/id/RebuildReplicatedIdGeneratorsTest.java index bf053a0af7dd8..f095b4d920af8 100644 --- a/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/core/state/machines/id/RebuildReplicatedIdGeneratorsTest.java +++ b/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/core/state/machines/id/RebuildReplicatedIdGeneratorsTest.java @@ -90,7 +90,6 @@ public void rebuildReplicatedIdGeneratorsOnRecovery() throws Exception reopenedStores.makeStoreOk(); assertEquals( 51L, reopenedStores.getNodeStore().nextId() ); } - } private ReplicatedIdGeneratorFactory getIdGenerationFactory( FileSystemAbstraction fileSystemAbstraction ) diff --git a/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/core/state/snapshot/CoreStateDownloaderTest.java b/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/core/state/snapshot/CoreStateDownloaderTest.java index 88c81547f060d..60f548a1bf84a 100644 --- a/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/core/state/snapshot/CoreStateDownloaderTest.java +++ b/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/core/state/snapshot/CoreStateDownloaderTest.java @@ -44,6 +44,7 @@ import static org.junit.Assert.fail; import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyBoolean; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.never; import static org.mockito.Mockito.verify; @@ -95,7 +96,7 @@ public void shouldDownloadCompleteStoreWhenEmpty() throws Throwable downloader.downloadSnapshot( remoteMember ); // then - verify( remoteStore, never() ).tryCatchingUp( any(), any(), any() ); + verify( remoteStore, never() ).tryCatchingUp( any(), any(), any(), anyBoolean() ); verify( storeCopyProcess ).replaceWithStoreFrom( remoteAddress, remoteStoreId ); } @@ -136,7 +137,7 @@ public void shouldNotOverwriteNonEmptyMismatchingStore() throws Exception // then verify( remoteStore, never() ).copy( any(), any(), any() ); - verify( remoteStore, never() ).tryCatchingUp( any(), any(), any() ); + verify( remoteStore, never() ).tryCatchingUp( any(), any(), any(), anyBoolean() ); } @Test @@ -145,13 +146,14 @@ public void shouldCatchupIfPossible() throws Exception // given when( localDatabase.isEmpty() ).thenReturn( false ); when( remoteStore.getStoreId( remoteAddress ) ).thenReturn( storeId ); - when( remoteStore.tryCatchingUp( remoteAddress, storeId, storeDir ) ).thenReturn( SUCCESS_END_OF_STREAM ); + when( remoteStore.tryCatchingUp( remoteAddress, storeId, storeDir, false ) ) + .thenReturn( SUCCESS_END_OF_STREAM ); // when downloader.downloadSnapshot( remoteMember ); // then - verify( remoteStore ).tryCatchingUp( remoteAddress, storeId, storeDir ); + verify( remoteStore ).tryCatchingUp( remoteAddress, storeId, storeDir, false ); verify( remoteStore, never() ).copy( any(), any(), any() ); } @@ -161,13 +163,13 @@ public void shouldDownloadWholeStoreIfCannotCatchUp() throws Exception // given when( localDatabase.isEmpty() ).thenReturn( false ); when( remoteStore.getStoreId( remoteAddress ) ).thenReturn( storeId ); - when( remoteStore.tryCatchingUp( remoteAddress, storeId, storeDir ) ).thenReturn( E_TRANSACTION_PRUNED ); + when( remoteStore.tryCatchingUp( remoteAddress, storeId, storeDir, false ) ).thenReturn( E_TRANSACTION_PRUNED ); // when downloader.downloadSnapshot( remoteMember ); // then - verify( remoteStore ).tryCatchingUp( remoteAddress, storeId, storeDir ); + verify( remoteStore ).tryCatchingUp( remoteAddress, storeId, storeDir, false ); verify( storeCopyProcess ).replaceWithStoreFrom( remoteAddress, storeId ); } } diff --git a/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/discovery/CoreClusterMember.java b/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/discovery/CoreClusterMember.java index 1b1d39d0f0255..ce3c86c6c2d50 100644 --- a/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/discovery/CoreClusterMember.java +++ b/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/discovery/CoreClusterMember.java @@ -67,6 +67,7 @@ public class CoreClusterMember implements ClusterMember private final String boltAdvertisedSocketAddress; private final int discoveryPort; protected CoreGraphDatabase database; + private final Config memberConfig; public CoreClusterMember( int serverId, int discoveryPort, @@ -124,12 +125,15 @@ public CoreClusterMember( int serverId, this.neo4jHome = new File( parentDir, "server-core-" + serverId ); config.put( GraphDatabaseSettings.neo4j_home.name(), neo4jHome.getAbsolutePath() ); config.put( GraphDatabaseSettings.logs_directory.name(), new File( neo4jHome, "logs" ).getAbsolutePath() ); + config.put( GraphDatabaseSettings.logical_logs_location.name(), "core-tx-logs-" + serverId ); this.discoveryServiceFactory = discoveryServiceFactory; File dataDir = new File( neo4jHome, "data" ); clusterStateDir = ClusterStateDirectory.withoutInitializing( dataDir ).get(); raftLogDir = new File( clusterStateDir, RAFT_LOG_DIRECTORY_NAME ); storeDir = new File( new File( dataDir, "databases" ), "graph.db" ); + memberConfig = Config.defaults( config ); + //noinspection ResultOfMethodCallIgnored storeDir.mkdirs(); } @@ -152,7 +156,7 @@ public String directURI() @Override public void start() { - database = new CoreGraphDatabase( storeDir, Config.defaults( config ), + database = new CoreGraphDatabase( storeDir, memberConfig, GraphDatabaseDependencies.newDependencies(), discoveryServiceFactory ); } @@ -177,6 +181,11 @@ public File storeDir() return storeDir; } + public Config getMemberConfig() + { + return memberConfig; + } + public RaftLogPruner raftLogPruner() { return database.getDependencyResolver().resolveDependency( RaftLogPruner.class ); diff --git a/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/discovery/ReadReplica.java b/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/discovery/ReadReplica.java index 8e6a9a8c186ac..39badd957129f 100644 --- a/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/discovery/ReadReplica.java +++ b/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/discovery/ReadReplica.java @@ -97,9 +97,11 @@ public ReadReplica( File parentDir, int serverId, int boltPort, int httpPort, in config.put( CausalClusteringSettings.transaction_listen_address.name(), listenAddress( listenAddress, txPort ) ); config.put( OnlineBackupSettings.online_backup_server.name(), listenAddress( listenAddress, backupPort ) ); config.put( GraphDatabaseSettings.logs_directory.name(), new File( neo4jHome, "logs" ).getAbsolutePath() ); + config.put( GraphDatabaseSettings.logical_logs_location.name(), "replica-tx-logs-" + serverId ); this.discoveryServiceFactory = discoveryServiceFactory; storeDir = new File( new File( new File( neo4jHome, "data" ), "databases" ), "graph.db" ); + //noinspection ResultOfMethodCallIgnored storeDir.mkdirs(); diff --git a/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/scenarios/ClusterCustomLogLocationIT.java b/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/scenarios/ClusterCustomLogLocationIT.java new file mode 100644 index 0000000000000..d00c37769cbbc --- /dev/null +++ b/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/scenarios/ClusterCustomLogLocationIT.java @@ -0,0 +1,99 @@ +/* + * Copyright (c) 2002-2017 "Neo Technology," + * Network Engine for Objects in Lund AB [http://neotechnology.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +package org.neo4j.causalclustering.scenarios; + +import org.hamcrest.Matchers; +import org.junit.Rule; +import org.junit.Test; + +import java.io.File; +import java.io.IOException; +import java.util.Collection; + +import org.neo4j.causalclustering.discovery.Cluster; +import org.neo4j.causalclustering.discovery.CoreClusterMember; +import org.neo4j.causalclustering.discovery.ReadReplica; +import org.neo4j.graphdb.DependencyResolver; +import org.neo4j.io.fs.FileSystemAbstraction; +import org.neo4j.kernel.impl.transaction.log.files.LogFiles; +import org.neo4j.test.causalclustering.ClusterRule; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertTrue; +import static org.neo4j.kernel.impl.transaction.log.files.LogFilesBuilder.logFilesBasedOnlyBuilder; + +public class ClusterCustomLogLocationIT +{ + @Rule + public final ClusterRule clusterRule = new ClusterRule( getClass() ) + .withNumberOfCoreMembers( 3 ) + .withNumberOfReadReplicas( 2 ); + + @Test + public void clusterWithCustomTransactionLogLocation() throws Exception + { + Cluster cluster = clusterRule.startCluster(); + + for ( int i = 0; i < 10; i++ ) + { + cluster.coreTx( ( db, tx ) -> + { + db.createNode(); + tx.success(); + } ); + } + + Collection coreClusterMembers = cluster.coreMembers(); + for ( CoreClusterMember coreClusterMember : coreClusterMembers ) + { + DependencyResolver dependencyResolver = coreClusterMember.database().getDependencyResolver(); + LogFiles logFiles = dependencyResolver.resolveDependency( LogFiles.class ); + assertEquals( logFiles.logFilesDirectory().getName(), "core-tx-logs-" + coreClusterMember.serverId() ); + assertTrue( logFiles.hasAnyEntries( 0 ) ); + File[] coreLogDirectories = coreClusterMember.storeDir().listFiles( file -> file.getName().startsWith( "core" ) ); + assertThat( coreLogDirectories, Matchers.arrayWithSize( 1 ) ); + + logFileInStoreDirectoryDoesNotExist( coreClusterMember.storeDir(), dependencyResolver ); + } + + Collection readReplicas = cluster.readReplicas(); + for ( ReadReplica readReplica : readReplicas ) + { + readReplica.txPollingClient().upToDateFuture().get(); + DependencyResolver dependencyResolver = readReplica.database().getDependencyResolver(); + LogFiles logFiles = dependencyResolver.resolveDependency( LogFiles.class ); + assertEquals( logFiles.logFilesDirectory().getName(), "replica-tx-logs-" + readReplica.serverId() ); + assertTrue( logFiles.hasAnyEntries( 0 ) ); + File[] replicaLogDirectories = readReplica.storeDir().listFiles( file -> file.getName().startsWith( "replica" ) ); + assertThat( replicaLogDirectories, Matchers.arrayWithSize( 1 ) ); + + logFileInStoreDirectoryDoesNotExist( readReplica.storeDir(), dependencyResolver ); + } + } + + private void logFileInStoreDirectoryDoesNotExist( File storeDir, DependencyResolver dependencyResolver ) throws IOException + { + FileSystemAbstraction fileSystem = dependencyResolver.resolveDependency( FileSystemAbstraction.class ); + LogFiles storeLogFiles = logFilesBasedOnlyBuilder( storeDir, fileSystem ).build(); + assertFalse( storeLogFiles.versionExists( 0 ) ); + } +} diff --git a/enterprise/com/src/main/java/org/neo4j/com/storecopy/FileMoveAction.java b/enterprise/com/src/main/java/org/neo4j/com/storecopy/FileMoveAction.java index a0468235cfe95..ec606137e8a44 100644 --- a/enterprise/com/src/main/java/org/neo4j/com/storecopy/FileMoveAction.java +++ b/enterprise/com/src/main/java/org/neo4j/com/storecopy/FileMoveAction.java @@ -29,33 +29,55 @@ import org.neo4j.io.fs.FileHandle; import org.neo4j.io.pagecache.PageCache; -@FunctionalInterface public interface FileMoveAction { void move( File toDir, CopyOption... copyOptions ) throws IOException; + File file(); + static FileMoveAction copyViaPageCache( File file, PageCache pageCache ) { - return ( toDir, copyOptions ) -> + return new FileMoveAction() { - Optional handle = pageCache.getCachedFileSystem().streamFilesRecursive( file ).findAny(); - if ( handle.isPresent() ) + + @Override + public void move( File toDir, CopyOption... copyOptions ) throws IOException + { + Optional handle = pageCache.getCachedFileSystem().streamFilesRecursive( file ).findAny(); + if ( handle.isPresent() ) + { + handle.get().rename( new File( toDir, file.getName() ), copyOptions ); + } + } + + @Override + public File file() { - handle.get().rename( new File( toDir, file.getName() ), copyOptions ); + return file; } }; } static FileMoveAction copyViaFileSystem( File file, File basePath ) { - Path base = basePath.toPath(); - return ( toDir, copyOptions ) -> + return new FileMoveAction() { - Path originalPath = file.toPath(); - Path relativePath = base.relativize( originalPath ); - Path resolvedPath = toDir.toPath().resolve( relativePath ); - Files.createDirectories( resolvedPath.getParent() ); - Files.copy( originalPath, resolvedPath, copyOptions ); + @Override + public void move( File toDir, CopyOption... copyOptions ) throws IOException + { + Path base = basePath.toPath(); + Path originalPath = file.toPath(); + Path relativePath = base.relativize( originalPath ); + Path resolvedPath = toDir.toPath().resolve( relativePath ); + Files.createDirectories( resolvedPath.getParent() ); + Files.copy( originalPath, resolvedPath, copyOptions ); + } + + @Override + public File file() + { + return file; + } }; } diff --git a/enterprise/com/src/main/java/org/neo4j/com/storecopy/MoveAfterCopy.java b/enterprise/com/src/main/java/org/neo4j/com/storecopy/MoveAfterCopy.java index 1563c047e7b05..7e37e493077a6 100644 --- a/enterprise/com/src/main/java/org/neo4j/com/storecopy/MoveAfterCopy.java +++ b/enterprise/com/src/main/java/org/neo4j/com/storecopy/MoveAfterCopy.java @@ -21,20 +21,23 @@ import java.io.File; import java.nio.file.StandardCopyOption; +import java.util.function.Function; import java.util.stream.Stream; +@FunctionalInterface public interface MoveAfterCopy { - void move( Stream moves, File fromDirectory, File toDirectory ) throws Exception; + void move( Stream moves, File fromDirectory, Function destinationFunction ) throws + Exception; static MoveAfterCopy moveReplaceExisting() { - return ( moves, fromDirectory, toDirectory ) -> + return ( moves, fromDirectory, destinationFunction ) -> { Iterable itr = moves::iterator; for ( FileMoveAction move : itr ) { - move.move( toDirectory, StandardCopyOption.REPLACE_EXISTING ); + move.move( destinationFunction.apply( move.file() ), StandardCopyOption.REPLACE_EXISTING ); } }; } diff --git a/enterprise/com/src/main/java/org/neo4j/com/storecopy/StoreCopyClient.java b/enterprise/com/src/main/java/org/neo4j/com/storecopy/StoreCopyClient.java index 3c2e2832969ed..72d549eb4880d 100644 --- a/enterprise/com/src/main/java/org/neo4j/com/storecopy/StoreCopyClient.java +++ b/enterprise/com/src/main/java/org/neo4j/com/storecopy/StoreCopyClient.java @@ -27,6 +27,7 @@ import java.util.Arrays; import java.util.List; import java.util.concurrent.atomic.AtomicLong; +import java.util.function.Function; import java.util.stream.Stream; import org.neo4j.com.Response; @@ -203,13 +204,18 @@ public void copyStore( StoreCopyRequester requester, CancellationRequest cancell graphDatabaseService.shutdown(); monitor.finishRecoveringStore(); + LogFiles logFiles = LogFilesBuilder.activeFilesBuilder( storeDir, fs, pageCache ) + .withConfig( config ) + .build(); // All is well, move the streamed files to the real store directory. // Start with the files written through the page cache. Should only be record store files. // Note that the stream is lazy, so the file system traversal won't happen until *after* the store files // have been moved. Thus we ensure that we only attempt to move them once. Stream moveActionStream = Stream.concat( storeFileMoveActions.stream(), traverseGenerateMoveActions( tempStore, tempStore ) ); - moveAfterCopy.move( moveActionStream, tempStore, storeDir ); + Function destinationMapper = + file -> logFiles.isLogFile( file ) ? logFiles.logFilesDirectory() : storeDir; + moveAfterCopy.move( moveActionStream, tempStore, destinationMapper ); } finally { @@ -333,6 +339,7 @@ private GraphDatabaseService newTempDatabase( File tempStore ) .setConfig( "dbms.backup.enabled", Settings.FALSE ) .setConfig( GraphDatabaseSettings.logs_directory, tempStore.getAbsolutePath() ) .setConfig( GraphDatabaseSettings.keep_logical_logs, Settings.TRUE ) + .setConfig( GraphDatabaseSettings.logical_logs_location, tempStore.getAbsolutePath() ) .setConfig( GraphDatabaseSettings.allow_upgrade, config.get( GraphDatabaseSettings.allow_upgrade ).toString() ) .newGraphDatabase(); diff --git a/enterprise/com/src/main/java/org/neo4j/com/storecopy/StoreCopyServer.java b/enterprise/com/src/main/java/org/neo4j/com/storecopy/StoreCopyServer.java index fbe03664cd80d..00dc64c9292dc 100644 --- a/enterprise/com/src/main/java/org/neo4j/com/storecopy/StoreCopyServer.java +++ b/enterprise/com/src/main/java/org/neo4j/com/storecopy/StoreCopyServer.java @@ -174,6 +174,7 @@ public RequestContext flushStoresAndStreamStoreFiles( String triggerName, StoreW { StoreFileMetadata meta = files.next(); File file = meta.file(); + boolean isLogFile = meta.isLogFile(); int recordSize = meta.recordSize(); // Read from paged file if mapping exists. Otherwise read through file system. @@ -187,7 +188,8 @@ public RequestContext flushStoresAndStreamStoreFiles( String triggerName, StoreW long fileSize = pagedFile.fileSize(); try ( ReadableByteChannel fileChannel = pagedFile.openReadableByteChannel() ) { - doWrite( writer, temporaryBuffer, file, recordSize, fileChannel, fileSize, storeCopyIdentifier ); + doWrite( writer, temporaryBuffer, file, recordSize, fileChannel, fileSize, + storeCopyIdentifier, false ); } } } @@ -196,7 +198,8 @@ public RequestContext flushStoresAndStreamStoreFiles( String triggerName, StoreW try ( ReadableByteChannel fileChannel = fileSystem.open( file, OpenMode.READ ) ) { long fileSize = fileSystem.getFileSize( file ); - doWrite( writer, temporaryBuffer, file, recordSize, fileChannel, fileSize, storeCopyIdentifier ); + doWrite( writer, temporaryBuffer, file, recordSize, fileChannel, fileSize, + storeCopyIdentifier, isLogFile ); } } } @@ -215,11 +218,11 @@ public RequestContext flushStoresAndStreamStoreFiles( String triggerName, StoreW } private void doWrite( StoreWriter writer, ByteBuffer temporaryBuffer, File file, int recordSize, - ReadableByteChannel fileChannel, long fileSize, String storeCopyIdentifier ) throws IOException + ReadableByteChannel fileChannel, long fileSize, String storeCopyIdentifier, boolean isLogFile ) throws IOException { monitor.startStreamingStoreFile( file, storeCopyIdentifier ); - writer.write( relativePath( storeDirectory, file ), fileChannel, - temporaryBuffer, fileSize > 0, recordSize ); + String path = isLogFile ? file.getName() : relativePath( storeDirectory, file ); + writer.write( path, fileChannel, temporaryBuffer, fileSize > 0, recordSize ); monitor.finishStreamingStoreFile( file, storeCopyIdentifier ); } } diff --git a/enterprise/com/src/test/java/org/neo4j/com/storecopy/StoreCopyClientTest.java b/enterprise/com/src/test/java/org/neo4j/com/storecopy/StoreCopyClientTest.java index b51322101ddc3..6e8d7c62621d5 100644 --- a/enterprise/com/src/test/java/org/neo4j/com/storecopy/StoreCopyClientTest.java +++ b/enterprise/com/src/test/java/org/neo4j/com/storecopy/StoreCopyClientTest.java @@ -52,6 +52,8 @@ import org.neo4j.kernel.impl.transaction.log.checkpoint.CheckPointer; import org.neo4j.kernel.impl.transaction.log.checkpoint.StoreCopyCheckPointMutex; import org.neo4j.kernel.impl.transaction.log.entry.LogHeader; +import org.neo4j.kernel.impl.transaction.log.files.LogFiles; +import org.neo4j.kernel.impl.transaction.log.files.LogFilesBuilder; import org.neo4j.kernel.internal.GraphDatabaseAPI; import org.neo4j.kernel.monitoring.Monitors; import org.neo4j.logging.NullLogProvider; @@ -62,28 +64,31 @@ import org.neo4j.test.rule.fs.DefaultFileSystemRule; import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.greaterThanOrEqualTo; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotEquals; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; import static org.mockito.Mockito.spy; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.neo4j.com.storecopy.StoreUtil.TEMP_COPY_DIRECTORY_NAME; import static org.neo4j.graphdb.Label.label; +import static org.neo4j.graphdb.factory.GraphDatabaseSettings.logical_logs_location; import static org.neo4j.graphdb.factory.GraphDatabaseSettings.record_format; public class StoreCopyClientTest { - private final TestDirectory testDir = TestDirectory.testDirectory(); + private final TestDirectory directory = TestDirectory.testDirectory(); private final PageCacheRule pageCacheRule = new PageCacheRule(); private final CleanupRule cleanup = new CleanupRule(); private final DefaultFileSystemRule fileSystemRule = new DefaultFileSystemRule(); @Rule - public TestRule rules = RuleChain.outerRule( testDir ).around( fileSystemRule ). + public TestRule rules = RuleChain.outerRule( directory ).around( fileSystemRule ). around( pageCacheRule ).around( cleanup ); private FileSystemAbstraction fileSystem; @@ -99,8 +104,8 @@ public void shouldCopyStoreFilesAcrossIfACancellationRequestHappensAfterTheTempS throws Exception { // given - final File copyDir = new File( testDir.directory(), "copy" ); - final File originalDir = new File( testDir.directory(), "original" ); + final File copyDir = new File( directory.directory(), "copy" ); + final File originalDir = new File( directory.directory(), "original" ); final AtomicBoolean cancelStoreCopy = new AtomicBoolean( false ); StoreCopyClient.Monitor storeCopyMonitor = new StoreCopyClient.Monitor.Adapter() @@ -129,7 +134,7 @@ public void finishRecoveringStore() } StoreCopyClient.StoreCopyRequester storeCopyRequest = - spy( new LocalStoreCopyRequester( original, originalDir, fileSystem ) ); + spy( new LocalStoreCopyRequester( original, originalDir, fileSystem, false ) ); // when copier.copyStore( storeCopyRequest, cancelStoreCopy::get, MoveAfterCopy.moveReplaceExisting() ); @@ -153,37 +158,58 @@ public void finishRecoveringStore() } @Test - public void storeCopyClientMustWorkWithStandardRecordFormat() throws Exception - { - checkStoreCopyClientWithRecordFormats( Standard.LATEST_NAME ); - } - - @Test - public void storeCopyClientMustWorkWithHighLimitRecordFormat() throws Exception + public void storeCopyClientUseCustomTransactionLogLocationWhenConfigured() throws Exception { - checkStoreCopyClientWithRecordFormats( HighLimit.NAME ); - } - - private void checkStoreCopyClientWithRecordFormats( String recordFormatsName ) throws Exception - { - final File copyDir = new File( testDir.directory(), "copy" ); - final File originalDir = new File( testDir.directory(), "original" ); + final File copyDir = new File( directory.directory(), "copyCustomLocation" ); + final File originalDir = new File( directory.directory(), "originalCustomLocation" ); PageCache pageCache = pageCacheRule.getPageCache( fileSystem ); - Config config = Config.defaults( record_format, recordFormatsName ); + File copyCustomLogFilesLocation = new File( copyDir, "CopyCustomLogFilesLocation" ); + File originalCustomLogFilesLocation = new File( originalDir, "originalCustomLogFilesLocation" ); + + Config config = Config.defaults( logical_logs_location, copyCustomLogFilesLocation.getName() ); StoreCopyClient copier = new StoreCopyClient( copyDir, config, loadKernelExtensions(), NullLogProvider.getInstance(), fileSystem, pageCache, new StoreCopyClient.Monitor.Adapter(), false ); - final GraphDatabaseAPI original = (GraphDatabaseAPI) startDatabase( originalDir, recordFormatsName ); + GraphDatabaseAPI original = (GraphDatabaseAPI) new TestGraphDatabaseFactory() + .newEmbeddedDatabaseBuilder( originalDir ) + .setConfig( logical_logs_location, originalCustomLogFilesLocation.getName() ) + .newGraphDatabase(); + generateTransactions( original ); + long logFileSize = + original.getDependencyResolver().resolveDependency( LogFiles.class ).getLogFileForVersion( 0 ).length(); + StoreCopyClient.StoreCopyRequester storeCopyRequest = new LocalStoreCopyRequester( original, originalDir, - fileSystem ); + fileSystem, true ); copier.copyStore( storeCopyRequest, CancellationRequest.NEVER_CANCELLED, MoveAfterCopy.moveReplaceExisting() ); + original.shutdown(); assertFalse( new File( copyDir, TEMP_COPY_DIRECTORY_NAME ).exists() ); - // Must not throw - startDatabase( copyDir, recordFormatsName ).shutdown(); + LogFiles customLogFiles = LogFilesBuilder.logFilesBasedOnlyBuilder( copyCustomLogFilesLocation, fileSystem ).build(); + assertTrue( customLogFiles.versionExists( 0 ) ); + assertThat( customLogFiles.getLogFileForVersion( 0 ).length(), greaterThanOrEqualTo( logFileSize ) ); + + LogFiles logFiles = LogFilesBuilder.logFilesBasedOnlyBuilder( copyDir, fileSystem ).build(); + assertFalse( logFiles.versionExists( 0 ) ); + + new TestGraphDatabaseFactory() + .newEmbeddedDatabaseBuilder( copyDir ) + .setConfig( logical_logs_location, copyCustomLogFilesLocation.getName() ) + .newGraphDatabase().shutdown(); + } + + @Test + public void storeCopyClientMustWorkWithStandardRecordFormat() throws Exception + { + checkStoreCopyClientWithRecordFormats( Standard.LATEST_NAME ); + } + + @Test + public void storeCopyClientMustWorkWithHighLimitRecordFormat() throws Exception + { + checkStoreCopyClientWithRecordFormats( HighLimit.NAME ); } @Test @@ -191,8 +217,8 @@ public void shouldEndUpWithAnEmptyStoreIfCancellationRequestIssuedJustBeforeReco throws Exception { // given - final File copyDir = new File( testDir.directory(), "copy" ); - final File originalDir = new File( testDir.directory(), "original" ); + final File copyDir = new File( directory.directory(), "copy" ); + final File originalDir = new File( directory.directory(), "original" ); final AtomicBoolean cancelStoreCopy = new AtomicBoolean( false ); StoreCopyClient.Monitor storeCopyMonitor = new StoreCopyClient.Monitor.Adapter() @@ -219,7 +245,7 @@ public void finishReceivingStoreFiles() } StoreCopyClient.StoreCopyRequester storeCopyRequest = - spy( new LocalStoreCopyRequester( original, originalDir, fileSystem ) ); + spy( new LocalStoreCopyRequester( original, originalDir, fileSystem, false ) ); // when copier.copyStore( storeCopyRequest, cancelStoreCopy::get, MoveAfterCopy.moveReplaceExisting() ); @@ -243,8 +269,8 @@ public void finishReceivingStoreFiles() public void shouldResetNeoStoreLastTransactionOffsetForNonForensicCopy() throws Exception { // GIVEN - File initialStore = testDir.directory( "initialStore" ); - File backupStore = testDir.directory( "backupStore" ); + File initialStore = directory.directory( "initialStore" ); + File backupStore = directory.directory( "backupStore" ); PageCache pageCache = pageCacheRule.getPageCache( fileSystem ); createInitialDatabase( initialStore ); @@ -259,7 +285,7 @@ public void shouldResetNeoStoreLastTransactionOffsetForNonForensicCopy() throws .getInstance(), fileSystem, pageCache, new StoreCopyClient.Monitor.Adapter(), false ); CancellationRequest falseCancellationRequest = () -> false; StoreCopyClient.StoreCopyRequester storeCopyRequest = - new LocalStoreCopyRequester( (GraphDatabaseAPI) initialDatabase, initialStore, fileSystem ); + new LocalStoreCopyRequester( (GraphDatabaseAPI) initialDatabase, initialStore, fileSystem, false ); // WHEN copier.copyStore( storeCopyRequest, falseCancellationRequest, MoveAfterCopy.moveReplaceExisting() ); @@ -277,8 +303,8 @@ public void shouldResetNeoStoreLastTransactionOffsetForNonForensicCopy() throws public void shouldDeleteTempCopyFolderOnFailures() throws Exception { // GIVEN - File initialStore = testDir.directory( "initialStore" ); - File backupStore = testDir.directory( "backupStore" ); + File initialStore = directory.directory( "initialStore" ); + File backupStore = directory.directory( "backupStore" ); PageCache pageCache = pageCacheRule.getPageCache( fileSystem ); GraphDatabaseService initialDatabase = createInitialDatabase( initialStore ); @@ -289,7 +315,7 @@ public void shouldDeleteTempCopyFolderOnFailures() throws Exception RuntimeException exception = new RuntimeException( "Boom!" ); StoreCopyClient.StoreCopyRequester storeCopyRequest = - new LocalStoreCopyRequester( (GraphDatabaseAPI) initialDatabase, initialStore, fileSystem ) + new LocalStoreCopyRequester( (GraphDatabaseAPI) initialDatabase, initialStore, fileSystem, false ) { @Override public void done() @@ -313,6 +339,28 @@ public void done() assertFalse( new File( backupStore, TEMP_COPY_DIRECTORY_NAME ).exists() ); } + private void checkStoreCopyClientWithRecordFormats( String recordFormatsName ) throws Exception + { + final File copyDir = new File( directory.directory(), "copy" ); + final File originalDir = new File( directory.directory(), "original" ); + PageCache pageCache = pageCacheRule.getPageCache( fileSystem ); + Config config = Config.defaults( record_format, recordFormatsName ); + StoreCopyClient copier = new StoreCopyClient( + copyDir, config, loadKernelExtensions(), NullLogProvider.getInstance(), fileSystem, pageCache, + new StoreCopyClient.Monitor.Adapter(), false ); + + final GraphDatabaseAPI original = (GraphDatabaseAPI) startDatabase( originalDir, recordFormatsName ); + StoreCopyClient.StoreCopyRequester storeCopyRequest = new LocalStoreCopyRequester( original, originalDir, + fileSystem, false ); + + copier.copyStore( storeCopyRequest, CancellationRequest.NEVER_CANCELLED, MoveAfterCopy.moveReplaceExisting() ); + + assertFalse( new File( copyDir, TEMP_COPY_DIRECTORY_NAME ).exists() ); + + // Must not throw + startDatabase( copyDir, recordFormatsName ).shutdown(); + } + private GraphDatabaseService createInitialDatabase( File initialStore ) { GraphDatabaseService initialDatabase = startDatabase( initialStore ); @@ -352,6 +400,18 @@ private static List> loadKernelExtensions() return kernelExtensions; } + private void generateTransactions( GraphDatabaseAPI original ) + { + for ( int i = 0; i < 10; i++ ) + { + try ( Transaction transaction = original.beginTx() ) + { + original.createNode(); + transaction.success(); + } + } + } + private static class LocalStoreCopyRequester implements StoreCopyClient.StoreCopyRequester { private final GraphDatabaseAPI original; @@ -359,12 +419,15 @@ private static class LocalStoreCopyRequester implements StoreCopyClient.StoreCop private final FileSystemAbstraction fs; private Response response; + private boolean includeLogs; - LocalStoreCopyRequester( GraphDatabaseAPI original, File originalDir, FileSystemAbstraction fs ) + LocalStoreCopyRequester( GraphDatabaseAPI original, File originalDir, FileSystemAbstraction fs, + boolean includeLogs ) { this.original = original; this.originalDir = originalDir; this.fs = fs; + this.includeLogs = includeLogs; } @Override @@ -388,7 +451,7 @@ public Response copyStore( StoreWriter writer ) RequestContext requestContext = new StoreCopyServer( neoStoreDataSource, checkPointer, fs, originalDir, new Monitors().newMonitor( StoreCopyServer.Monitor.class ), pageCache, new StoreCopyCheckPointMutex() ) - .flushStoresAndStreamStoreFiles( "test", writer, false ); + .flushStoresAndStreamStoreFiles( "test", writer, includeLogs ); final StoreId storeId = original.getDependencyResolver().resolveDependency( RecordStorageEngine.class ) diff --git a/enterprise/fulltext-addon/src/test/java/org/neo4j/kernel/api/impl/fulltext/integrations/bloom/BloomIT.java b/enterprise/fulltext-addon/src/test/java/org/neo4j/kernel/api/impl/fulltext/integrations/bloom/BloomIT.java index 2c272851cdd52..100bb484c4fc6 100644 --- a/enterprise/fulltext-addon/src/test/java/org/neo4j/kernel/api/impl/fulltext/integrations/bloom/BloomIT.java +++ b/enterprise/fulltext-addon/src/test/java/org/neo4j/kernel/api/impl/fulltext/integrations/bloom/BloomIT.java @@ -19,7 +19,6 @@ */ package org.neo4j.kernel.api.impl.fulltext.integrations.bloom; -import org.apache.commons.lang3.SystemUtils; import org.apache.lucene.analysis.en.EnglishAnalyzer; import org.apache.lucene.analysis.sv.SwedishAnalyzer; import org.junit.After; @@ -28,7 +27,6 @@ import org.junit.Test; import org.junit.rules.ExpectedException; -import java.io.File; import java.util.Date; import org.neo4j.consistency.ConsistencyCheckService; @@ -57,8 +55,6 @@ import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertThat; import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; -import static org.junit.Assume.assumeFalse; import static org.neo4j.kernel.api.impl.fulltext.integrations.bloom.BloomFulltextConfig.bloom_enabled; public class BloomIT @@ -89,8 +85,8 @@ public class BloomIT public void before() throws Exception { GraphDatabaseFactory factory = new GraphDatabaseFactory(); - builder = factory.newEmbeddedDatabaseBuilder( testDirectory.graphDbDir() ); - builder.setConfig( bloom_enabled, "true" ); + builder = factory.newEmbeddedDatabaseBuilder( testDirectory.graphDbDir() ) + .setConfig( bloom_enabled, "true" ); } @After @@ -604,58 +600,4 @@ public void onlineIndexShouldBeReportedAsOnline() throws Exception assertEquals( "ONLINE", result.next().get( "state" ) ); assertFalse( result.hasNext() ); } - - @Test - public void failureToStartUpMustNotPreventShutDown() throws Exception - { - // Ignore this test on Windows because the test relies on file permissions to trigger failure modes in - // the code. Unfortunately, file permissions are an incredible pain to work with on Windows. - assumeFalse( SystemUtils.IS_OS_WINDOWS ); - - // Create the store directory and all its files, and add a bit of data to it - GraphDatabaseService db = getDb(); - db.execute( String.format( SET_NODE_KEYS, "\"prop\"" ) ); - - try ( Transaction tx = db.beginTx() ) - { - db.createNode().setProperty( "prop", "bla bla bla" ); - tx.success(); - } - db.shutdown(); - - File dir = testDirectory.graphDbDir(); - assertTrue( dir.setReadable( false ) ); - try - { - // Making the directory not readable ought to cause problems for the database as it tries to start up - getDb().shutdown(); - fail( "Should not have started up and shut down cleanly on an unreadable store directory" ); - } - catch ( Exception e ) - { - // Good - } - catch ( Throwable th ) - { - makeReadable( dir, th ); - throw th; - } - makeReadable( dir, null ); - } - - private void makeReadable( File dir, Throwable th ) - { - if ( !dir.setReadable( true ) ) - { - AssertionError error = new AssertionError( "Failed to make " + dir + " writable again!" ); - if ( th != null ) - { - th.addSuppressed( error ); - } - else - { - throw error; - } - } } -} diff --git a/enterprise/ha/src/test/java/org/neo4j/kernel/ha/cluster/SwitchToSlaveCopyThenBranchTest.java b/enterprise/ha/src/test/java/org/neo4j/kernel/ha/cluster/SwitchToSlaveCopyThenBranchTest.java index 1717f3d7b591a..8d5667e1da732 100644 --- a/enterprise/ha/src/test/java/org/neo4j/kernel/ha/cluster/SwitchToSlaveCopyThenBranchTest.java +++ b/enterprise/ha/src/test/java/org/neo4j/kernel/ha/cluster/SwitchToSlaveCopyThenBranchTest.java @@ -29,6 +29,7 @@ import java.net.URI; import java.net.URISyntaxException; import java.util.concurrent.TimeUnit; +import java.util.function.Function; import java.util.function.Predicate; import java.util.stream.Stream; @@ -231,7 +232,7 @@ public void shouldNotBranchStoreUnlessWeHaveCopiedDownAReplacement() throws Thro doAnswer( invocation -> { MoveAfterCopy moveAfterCopy = invocation.getArgument( 2 ); - moveAfterCopy.move( Stream.empty(), new File( "" ), new File( "" ) ); + moveAfterCopy.move( Stream.empty(), new File( "" ), Function.identity() ); return null; } ).when( storeCopyClient ).copyStore( any( StoreCopyClient.StoreCopyRequester.class ), diff --git a/enterprise/kernel/src/test/java/org/neo4j/util/TestHelpers.java b/enterprise/kernel/src/test/java/org/neo4j/util/TestHelpers.java index 18705a3143dcd..3593ce665f16b 100644 --- a/enterprise/kernel/src/test/java/org/neo4j/util/TestHelpers.java +++ b/enterprise/kernel/src/test/java/org/neo4j/util/TestHelpers.java @@ -25,22 +25,16 @@ import java.util.List; import org.neo4j.commandline.admin.AdminTool; -import org.neo4j.helpers.AdvertisedSocketAddress; import org.neo4j.helpers.HostnamePort; import org.neo4j.helpers.ListenSocketAddress; import org.neo4j.io.proc.ProcessUtil; import org.neo4j.kernel.configuration.Config; -import org.neo4j.kernel.configuration.Settings; import org.neo4j.kernel.impl.enterprise.configuration.OnlineBackupSettings; -import org.neo4j.kernel.impl.factory.GraphDatabaseFacade; import org.neo4j.kernel.internal.GraphDatabaseAPI; import org.neo4j.test.ProcessStreamHandler; -import org.neo4j.test.rule.DatabaseRule; -import org.neo4j.test.rule.EmbeddedDatabaseRule; import static java.lang.String.format; import static org.neo4j.kernel.configuration.Settings.listenAddress; -import static org.neo4j.kernel.configuration.Settings.setting; public class TestHelpers { diff --git a/enterprise/server-enterprise/src/main/java/org/neo4j/server/enterprise/EnterpriseNeoServer.java b/enterprise/server-enterprise/src/main/java/org/neo4j/server/enterprise/EnterpriseNeoServer.java index 58b60b9aa5a6d..51442ac6646b3 100644 --- a/enterprise/server-enterprise/src/main/java/org/neo4j/server/enterprise/EnterpriseNeoServer.java +++ b/enterprise/server-enterprise/src/main/java/org/neo4j/server/enterprise/EnterpriseNeoServer.java @@ -30,15 +30,15 @@ import org.neo4j.causalclustering.core.CausalClusteringSettings; import org.neo4j.causalclustering.core.CoreGraphDatabase; import org.neo4j.causalclustering.readreplica.ReadReplicaGraphDatabase; -import org.neo4j.dbms.DatabaseManagementSystemSettings; +import org.neo4j.graphdb.factory.GraphDatabaseSettings; import org.neo4j.helpers.collection.Iterables; import org.neo4j.kernel.configuration.Config; import org.neo4j.kernel.enterprise.EnterpriseGraphDatabase; import org.neo4j.kernel.ha.HaSettings; import org.neo4j.kernel.ha.HighlyAvailableGraphDatabase; -import org.neo4j.kernel.impl.factory.GraphDatabaseFacadeFactory; import org.neo4j.kernel.impl.enterprise.configuration.EnterpriseEditionSettings; import org.neo4j.kernel.impl.enterprise.configuration.EnterpriseEditionSettings.Mode; +import org.neo4j.kernel.impl.factory.GraphDatabaseFacadeFactory; import org.neo4j.kernel.impl.factory.GraphDatabaseFacadeFactory.Dependencies; import org.neo4j.kernel.impl.util.UnsatisfiedDependencyException; import org.neo4j.logging.LogProvider; @@ -65,25 +65,25 @@ public class EnterpriseNeoServer extends CommunityNeoServer private static final GraphFactory HA_FACTORY = ( config, dependencies ) -> { - File storeDir = config.get( DatabaseManagementSystemSettings.database_path ); + File storeDir = config.get( GraphDatabaseSettings.database_path ); return new HighlyAvailableGraphDatabase( storeDir, config, dependencies ); }; private static final GraphFactory ENTERPRISE_FACTORY = ( config, dependencies ) -> { - File storeDir = config.get( DatabaseManagementSystemSettings.database_path ); + File storeDir = config.get( GraphDatabaseSettings.database_path ); return new EnterpriseGraphDatabase( storeDir, config, dependencies ); }; private static final GraphFactory CORE_FACTORY = ( config, dependencies ) -> { - File storeDir = config.get( DatabaseManagementSystemSettings.database_path ); + File storeDir = config.get( GraphDatabaseSettings.database_path ); return new CoreGraphDatabase( storeDir, config, dependencies ); }; private static final GraphFactory READ_REPLICA_FACTORY = ( config, dependencies ) -> { - File storeDir = config.get( DatabaseManagementSystemSettings.database_path ); + File storeDir = config.get( GraphDatabaseSettings.database_path ); return new ReadReplicaGraphDatabase( storeDir, config, dependencies ); }; diff --git a/enterprise/server-enterprise/src/test/java/org/neo4j/server/enterprise/EnterpriseBootstrapperTestIT.java b/enterprise/server-enterprise/src/test/java/org/neo4j/server/enterprise/EnterpriseBootstrapperTestIT.java index 0fbd2241d1911..9053deed07e87 100644 --- a/enterprise/server-enterprise/src/test/java/org/neo4j/server/enterprise/EnterpriseBootstrapperTestIT.java +++ b/enterprise/server-enterprise/src/test/java/org/neo4j/server/enterprise/EnterpriseBootstrapperTestIT.java @@ -45,7 +45,7 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; -import static org.neo4j.dbms.DatabaseManagementSystemSettings.data_directory; +import static org.neo4j.graphdb.factory.GraphDatabaseSettings.data_directory; import static org.neo4j.graphdb.factory.GraphDatabaseSettings.logs_directory; import static org.neo4j.graphdb.factory.GraphDatabaseSettings.store_internal_log_level; import static org.neo4j.helpers.collection.MapUtil.store; diff --git a/enterprise/server-enterprise/src/test/java/org/neo4j/server/enterprise/jmx/ServerManagementIT.java b/enterprise/server-enterprise/src/test/java/org/neo4j/server/enterprise/jmx/ServerManagementIT.java index 5c47836547e62..fe71fb52dade1 100644 --- a/enterprise/server-enterprise/src/test/java/org/neo4j/server/enterprise/jmx/ServerManagementIT.java +++ b/enterprise/server-enterprise/src/test/java/org/neo4j/server/enterprise/jmx/ServerManagementIT.java @@ -23,7 +23,6 @@ import org.junit.Test; import org.junit.rules.RuleChain; -import org.neo4j.dbms.DatabaseManagementSystemSettings; import org.neo4j.graphdb.factory.GraphDatabaseSettings; import org.neo4j.kernel.GraphDatabaseDependencies; import org.neo4j.kernel.configuration.Config; @@ -70,17 +69,17 @@ public void shouldBeAbleToRestartServer() throws Exception server.start(); assertNotNull( server.getDatabase().getGraph() ); - assertEquals( config.get( DatabaseManagementSystemSettings.database_path ).getAbsolutePath(), + assertEquals( config.get( GraphDatabaseSettings.database_path ).getAbsolutePath(), server.getDatabase().getLocation().getAbsolutePath() ); // Change the database location - config.augment( DatabaseManagementSystemSettings.data_directory, dataDirectory2 ); + config.augment( GraphDatabaseSettings.data_directory, dataDirectory2 ); ServerManagement bean = new ServerManagement( server ); bean.restartServer(); // Then assertNotNull( server.getDatabase().getGraph() ); - assertEquals( config.get( DatabaseManagementSystemSettings.database_path ).getAbsolutePath(), + assertEquals( config.get( GraphDatabaseSettings.database_path ).getAbsolutePath(), server.getDatabase().getLocation().getAbsolutePath() ); } diff --git a/integrationtests/src/test/java/org/neo4j/storeupgrade/StoreUpgradeIT.java b/integrationtests/src/test/java/org/neo4j/storeupgrade/StoreUpgradeIT.java index 68f5bb8b34475..4b16e3ba21f6b 100644 --- a/integrationtests/src/test/java/org/neo4j/storeupgrade/StoreUpgradeIT.java +++ b/integrationtests/src/test/java/org/neo4j/storeupgrade/StoreUpgradeIT.java @@ -39,7 +39,6 @@ import java.util.Properties; import org.neo4j.backup.OnlineBackupSettings; -import org.neo4j.dbms.DatabaseManagementSystemSettings; import org.neo4j.graphdb.GraphDatabaseService; import org.neo4j.graphdb.Label; import org.neo4j.graphdb.Node; @@ -49,12 +48,12 @@ import org.neo4j.graphdb.factory.GraphDatabaseSettings; import org.neo4j.helpers.Exceptions; import org.neo4j.helpers.collection.Iterables; +import org.neo4j.internal.kernel.api.exceptions.KernelException; import org.neo4j.io.fs.FileUtils; import org.neo4j.kernel.api.InwardKernel; import org.neo4j.kernel.api.KernelTransaction; import org.neo4j.kernel.api.ReadOperations; import org.neo4j.kernel.api.Statement; -import org.neo4j.internal.kernel.api.exceptions.KernelException; import org.neo4j.kernel.api.schema.index.IndexDescriptor; import org.neo4j.kernel.api.security.AnonymousContext; import org.neo4j.kernel.configuration.BoltConnector; @@ -171,15 +170,15 @@ public void embeddedDatabaseShouldStartOnOlderStoreWhenUpgradeIsEnabled() throws public void serverDatabaseShouldStartOnOlderStoreWhenUpgradeIsEnabled() throws Throwable { File rootDir = testDir.directory(); - File storeDir = Config.defaults( DatabaseManagementSystemSettings.data_directory, rootDir.toString() ) - .get( DatabaseManagementSystemSettings.database_path ); + File storeDir = Config.defaults( GraphDatabaseSettings.data_directory, rootDir.toString() ) + .get( GraphDatabaseSettings.database_path ); store.prepareDirectory( storeDir ); File configFile = new File( rootDir, Config.DEFAULT_CONFIG_FILE_NAME ); Properties props = new Properties(); props.putAll( ServerTestUtils.getDefaultRelativeProperties() ); - props.setProperty( DatabaseManagementSystemSettings.data_directory.name(), rootDir.getAbsolutePath() ); + props.setProperty( GraphDatabaseSettings.data_directory.name(), rootDir.getAbsolutePath() ); props.setProperty( GraphDatabaseSettings.logs_directory.name(), rootDir.getAbsolutePath() ); props.setProperty( GraphDatabaseSettings.allow_upgrade.name(), "true" ); props.setProperty( GraphDatabaseSettings.pagecache_memory.name(), "8m" ); diff --git a/tools/src/main/java/org/neo4j/tools/applytx/ApplyTransactionsCommand.java b/tools/src/main/java/org/neo4j/tools/applytx/ApplyTransactionsCommand.java index 06bb7ad984ad2..b1732d3837347 100644 --- a/tools/src/main/java/org/neo4j/tools/applytx/ApplyTransactionsCommand.java +++ b/tools/src/main/java/org/neo4j/tools/applytx/ApplyTransactionsCommand.java @@ -33,6 +33,7 @@ import org.neo4j.io.pagecache.PageCache; import org.neo4j.io.pagecache.impl.muninn.StandalonePageCacheFactory; import org.neo4j.kernel.api.exceptions.TransactionFailureException; +import org.neo4j.kernel.configuration.Config; import org.neo4j.kernel.impl.api.TransactionRepresentationCommitProcess; import org.neo4j.kernel.impl.api.TransactionToApply; import org.neo4j.kernel.impl.transaction.CommittedTransactionRepresentation; @@ -66,8 +67,9 @@ public ApplyTransactionsCommand( File from, Supplier to ) @Override protected void run( Args args, PrintStream out ) throws Exception { - TransactionIdStore txIdStore = to.get().getDependencyResolver().resolveDependency( - TransactionIdStore.class ); + DependencyResolver dependencyResolver = to.get().getDependencyResolver(); + TransactionIdStore txIdStore = dependencyResolver.resolveDependency( TransactionIdStore.class ); + Config config = dependencyResolver.resolveDependency( Config.class ); long fromTx = txIdStore.getLastCommittedTransaction().transactionId(); long toTx; if ( args.orphans().isEmpty() ) @@ -89,12 +91,12 @@ else if ( whereTo.equals( "last" ) ) toTx = Long.parseLong( whereTo ); } - long lastApplied = applyTransactions( from, to.get(), fromTx, toTx, out ); + long lastApplied = applyTransactions( from, to.get(), config, fromTx, toTx, out ); out.println( "Applied transactions up to and including " + lastApplied ); } - private long applyTransactions( File fromPath, GraphDatabaseAPI toDb, long fromTxExclusive, long toTxInclusive, - PrintStream out ) + private long applyTransactions( File fromPath, GraphDatabaseAPI toDb, Config toConfig, + long fromTxExclusive, long toTxInclusive, PrintStream out ) throws IOException, TransactionFailureException { DependencyResolver resolver = toDb.getDependencyResolver(); @@ -107,7 +109,7 @@ private long applyTransactions( File fromPath, GraphDatabaseAPI toDb, long fromT PageCache pageCache = StandalonePageCacheFactory.createPageCache( fileSystem ) ) { LogicalTransactionStore source = life.add( new ReadOnlyTransactionStore( pageCache, fileSystem, fromPath, - new Monitors() ) ); + Config.defaults(), new Monitors() ) ); life.start(); long lastAppliedTx = fromTxExclusive; // Some progress if there are more than a couple of transactions to apply diff --git a/tools/src/main/java/org/neo4j/tools/migration/StoreMigration.java b/tools/src/main/java/org/neo4j/tools/migration/StoreMigration.java index 7a982b4d47dd4..bdd142f33eef0 100644 --- a/tools/src/main/java/org/neo4j/tools/migration/StoreMigration.java +++ b/tools/src/main/java/org/neo4j/tools/migration/StoreMigration.java @@ -131,7 +131,8 @@ public void run( final FileSystemAbstraction fs, final File storeDirectory, Conf kernelContext, GraphDatabaseDependencies.newDependencies().kernelExtensions(), deps, ignore() ) ); - final LogFiles logFiles = LogFilesBuilder.activeFilesBuilder( storeDirectory, fs, pageCache ).build(); + final LogFiles logFiles = LogFilesBuilder.activeFilesBuilder( storeDirectory, fs, pageCache ) + .withConfig( config ).build(); LogTailScanner tailScanner = new LogTailScanner( logFiles, new VersionAwareLogEntryReader<>(), monitors ); // Add the kernel store migrator