Allow transaction logs to be located in a separate folder.

Previously transaction logs were always located in the same directory as a store.
This PR will allow changing that by specifying "logical_logs_location"
with a path to a folder where transaction logs should be stored.
This will allow placing transaction logs on a separate dedicated drive.

CC integration tests will always put transaction logs in separate directories.
HA should be able to copy store from another instance with custom
transactional logs directory and place them into the correct directory
for that specific instance.
Backup will put transaction logs together with store files and restore command will properly put them into configured transaction logs folder

community/command-line/src/main/java/org/neo4j/commandline/Util.java

@@ -63,6 +63,20 @@ public static Path canonicalPath( File file ) throws IllegalArgumentException
         }
     }
 
+    public static boolean isSameOrChildFile( File parent, File candidate )
+    {
+        Path canonicalCandidate = canonicalPath( candidate );
+        Path canonicalParentPath = canonicalPath( parent );
+        return canonicalCandidate.startsWith( canonicalParentPath );
+    }
+
+    public static boolean isSameOrChildPath( Path parent, Path candidate )
+    {
+        Path normalizedCandidate = candidate.normalize();
+        Path normalizedParent = parent.normalize();
+        return normalizedCandidate.startsWith( normalizedParent );
+    }
+
     public static void checkLock( Path databaseDirectory ) throws CommandFailed
     {
         try ( FileSystemAbstraction fileSystem = new DefaultFileSystemAbstraction();

community/consistency-check/src/main/java/org/neo4j/consistency/CheckConsistencyCommand.java

@@ -36,7 +36,6 @@
 import org.neo4j.commandline.arguments.common.OptionalCanonicalPath;
 import org.neo4j.consistency.checking.full.ConsistencyCheckIncompleteException;
 import org.neo4j.consistency.checking.full.ConsistencyFlags;
-import org.neo4j.dbms.DatabaseManagementSystemSettings;
 import org.neo4j.graphdb.factory.GraphDatabaseSettings;
 import org.neo4j.helpers.Strings;
 import org.neo4j.helpers.collection.MapUtil;
@@ -51,7 +50,7 @@
 import org.neo4j.logging.FormattedLogProvider;
 
 import static java.lang.String.format;
-import static org.neo4j.dbms.DatabaseManagementSystemSettings.database_path;
+import static org.neo4j.graphdb.factory.GraphDatabaseSettings.database_path;
 
 public class CheckConsistencyCommand implements AdminCommand
 {
@@ -234,7 +233,7 @@ private void checkDbState( File storeDir, Config additionalConfiguration ) throw
                       .createPageCache( fileSystem, additionalConfiguration ) )
         {
             RecoveryRequiredChecker requiredChecker =
-                    new RecoveryRequiredChecker( fileSystem, pageCache, new Monitors() );
+                    new RecoveryRequiredChecker( fileSystem, pageCache, additionalConfiguration, new Monitors() );
             if ( requiredChecker.isRecoveryRequiredAt( storeDir ) )
             {
                 throw new CommandFailed(
@@ -253,7 +252,7 @@ private void checkDbState( File storeDir, Config additionalConfiguration ) throw
     private static Config loadNeo4jConfig( Path homeDir, Path configDir, String databaseName,
             Map<String,String> additionalConfig )
     {
-        additionalConfig.put( DatabaseManagementSystemSettings.active_database.name(), databaseName );
+        additionalConfig.put( GraphDatabaseSettings.active_database.name(), databaseName );
 
         return Config.fromFile( configDir.resolve( Config.DEFAULT_CONFIG_FILE_NAME ) ).withHome( homeDir ).withConnectorsDisabled()
                 .withSettings( additionalConfig ).build();

community/consistency-check/src/main/java/org/neo4j/consistency/ConsistencyCheckTool.java

@@ -26,8 +26,8 @@
 import java.util.List;
 import java.util.Map;
 
-import org.neo4j.consistency.checking.full.ConsistencyFlags;
 import org.neo4j.consistency.checking.full.ConsistencyCheckIncompleteException;
+import org.neo4j.consistency.checking.full.ConsistencyFlags;
 import org.neo4j.graphdb.factory.GraphDatabaseSettings;
 import org.neo4j.helpers.Args;
 import org.neo4j.helpers.Strings;
@@ -137,7 +137,8 @@ private void checkDbState( File storeDir, Config tuningConfiguration ) throws To
     {
         try ( PageCache pageCache = ConfigurableStandalonePageCacheFactory.createPageCache( fs, tuningConfiguration ) )
         {
-            RecoveryRequiredChecker requiredChecker = new RecoveryRequiredChecker( fs, pageCache, new Monitors() );
+            RecoveryRequiredChecker requiredChecker = new RecoveryRequiredChecker( fs, pageCache,
+                    tuningConfiguration, new Monitors() );
             if ( requiredChecker.isRecoveryRequiredAt( storeDir ) )
             {
                 throw new ToolFailureException( Strings.joinAsLines(

community/consistency-check/src/test/java/org/neo4j/consistency/ConsistencyCheckToolTest.java

@@ -43,6 +43,7 @@
 import org.neo4j.graphdb.GraphDatabaseService;
 import org.neo4j.graphdb.Transaction;
 import org.neo4j.graphdb.factory.GraphDatabaseSettings;
+import org.neo4j.helpers.collection.MapUtil;
 import org.neo4j.helpers.progress.ProgressMonitorFactory;
 import org.neo4j.io.fs.DefaultFileSystemAbstraction;
 import org.neo4j.io.fs.FileSystemAbstraction;
@@ -66,20 +67,21 @@
 import static org.mockito.Mockito.verify;
 import static org.mockito.Mockito.verifyZeroInteractions;
 import static org.neo4j.graphdb.Label.label;
+import static org.neo4j.graphdb.factory.GraphDatabaseSettings.logical_logs_location;
 
 public class ConsistencyCheckToolTest
 {
-    private final TestDirectory storeDirectory = TestDirectory.testDirectory();
+    private final TestDirectory testDirectory = TestDirectory.testDirectory();
     private final EphemeralFileSystemRule fs = new EphemeralFileSystemRule();
 
     @Rule
-    public RuleChain ruleChain = RuleChain.outerRule( storeDirectory ).around( fs );
+    public RuleChain ruleChain = RuleChain.outerRule( testDirectory ).around( fs );
 
     @Test
     public void runsConsistencyCheck() throws Exception
     {
         // given
-        File storeDir = storeDirectory.directory();
+        File storeDir = testDirectory.directory();
         String[] args = {storeDir.getPath()};
         ConsistencyCheckService service = mock( ConsistencyCheckService.class );
 
@@ -108,8 +110,8 @@ public void consistencyCheckerLogUseSystemTimezoneIfConfigurable() throws Except
                         provider.getLog( "test" ).info( "testMessage" );
                         return ConsistencyCheckService.Result.success( new File( StringUtils.EMPTY ) );
                     } );
-            File storeDir = storeDirectory.directory();
-            File configFile = storeDirectory.file( Config.DEFAULT_CONFIG_FILE_NAME );
+            File storeDir = testDirectory.directory();
+            File configFile = testDirectory.file( Config.DEFAULT_CONFIG_FILE_NAME );
             Properties properties = new Properties();
             properties.setProperty( GraphDatabaseSettings.log_timezone.name(), LogTimeZone.SYSTEM.name() );
             properties.store( new FileWriter( configFile ), null );
@@ -128,7 +130,7 @@ public void consistencyCheckerLogUseSystemTimezoneIfConfigurable() throws Except
     public void appliesDefaultTuningConfigurationForConsistencyChecker() throws Exception
     {
         // given
-        File storeDir = storeDirectory.directory();
+        File storeDir = testDirectory.directory();
         String[] args = {storeDir.getPath()};
         ConsistencyCheckService service = mock( ConsistencyCheckService.class );
 
@@ -147,8 +149,8 @@ public void appliesDefaultTuningConfigurationForConsistencyChecker() throws Exce
     public void passesOnConfigurationIfProvided() throws Exception
     {
         // given
-        File storeDir = storeDirectory.directory();
-        File configFile = storeDirectory.file( Config.DEFAULT_CONFIG_FILE_NAME );
+        File storeDir = testDirectory.directory();
+        File configFile = testDirectory.file( Config.DEFAULT_CONFIG_FILE_NAME );
         Properties properties = new Properties();
         properties.setProperty( ConsistencyCheckSettings.consistency_check_property_owners.name(), "true" );
         properties.store( new FileWriter( configFile ), null );
@@ -191,8 +193,8 @@ public void exitWithFailureIndicatingCorrectUsageIfNoArgumentsSupplied() throws
     public void exitWithFailureIfConfigSpecifiedButConfigFileDoesNotExist() throws Exception
     {
         // given
-        File configFile = storeDirectory.file( "nonexistent_file" );
-        String[] args = {storeDirectory.directory().getPath(), "-config", configFile.getPath()};
+        File configFile = testDirectory.file( "nonexistent_file" );
+        String[] args = {testDirectory.directory().getPath(), "-config", configFile.getPath()};
         ConsistencyCheckService service = mock( ConsistencyCheckService.class );
 
         try
@@ -214,9 +216,34 @@ public void exitWithFailureIfConfigSpecifiedButConfigFileDoesNotExist() throws E
     @Test( expected = ToolFailureException.class )
     public void failWhenStoreWasNonCleanlyShutdown() throws Exception
     {
-        createGraphDbAndKillIt();
+        createGraphDbAndKillIt( Config.defaults() );
 
-        runConsistencyCheckToolWith( fs.get(), storeDirectory.graphDbDir().getAbsolutePath() );
+        runConsistencyCheckToolWith( fs.get(), testDirectory.graphDbDir().getAbsolutePath() );
+    }
+
+    @Test( expected = ToolFailureException.class )
+    public void failOnNotCleanlyShutdownStoreWithLogsInCustomRelativeLocation() throws Exception
+    {
+        File customConfigFile = testDirectory.file( "customConfig" );
+        Config customConfig = Config.defaults( logical_logs_location, "otherLocation" );
+        createGraphDbAndKillIt( customConfig );
+        MapUtil.store( customConfig.getRaw(), customConfigFile );
+        String[] args = {testDirectory.graphDbDir().getPath(), "-config", customConfigFile.getPath()};
+
+        runConsistencyCheckToolWith( fs.get(), args );
+    }
+
+    @Test( expected = ToolFailureException.class )
+    public void failOnNotCleanlyShutdownStoreWithLogsInCustomAbsoluteLocation() throws Exception
+    {
+        File customConfigFile = testDirectory.file( "customConfig" );
+        File otherLocation = testDirectory.directory( "otherLocation" );
+        Config customConfig = Config.defaults( logical_logs_location, otherLocation.getAbsolutePath() );
+        createGraphDbAndKillIt( customConfig );
+        MapUtil.store( customConfig.getRaw(), customConfigFile );
+        String[] args = {testDirectory.graphDbDir().getPath(), "-config", customConfigFile.getPath()};
+
+        runConsistencyCheckToolWith( fs.get(), args );
     }
 
     private void checkLogRecordTimeZone( ConsistencyCheckService service, String[] args, int hoursShift,
@@ -237,11 +264,12 @@ private String readLogLine( ByteArrayOutputStream outputStream ) throws IOExcept
         return bufferedReader.readLine();
     }
 
-    private void createGraphDbAndKillIt() throws Exception
+    private void createGraphDbAndKillIt( Config config ) throws Exception
     {
         final GraphDatabaseService db = new TestGraphDatabaseFactory()
                 .setFileSystem( fs.get() )
-                .newImpermanentDatabaseBuilder( storeDirectory.graphDbDir() )
+                .newImpermanentDatabaseBuilder( testDirectory.graphDbDir() )
+                .setConfig( config.getRaw()  )
                 .newGraphDatabase();
 
         try ( Transaction tx = db.beginTx() )

community/dbms/src/main/java/org/neo4j/commandline/dbms/CsvImporter.java

@@ -30,7 +30,6 @@
 import org.neo4j.commandline.admin.OutsideWorld;
 import org.neo4j.commandline.dbms.config.WrappedBatchImporterConfigurationForNeo4jAdmin;
 import org.neo4j.commandline.dbms.config.WrappedCsvInputConfigurationForNeo4jAdmin;
-import org.neo4j.dbms.DatabaseManagementSystemSettings;
 import org.neo4j.graphdb.factory.GraphDatabaseSettings;
 import org.neo4j.helpers.Args;
 import org.neo4j.io.fs.FileSystemAbstraction;
@@ -43,7 +42,6 @@
 import org.neo4j.unsafe.impl.batchimport.input.csv.IdType;
 
 import static java.nio.charset.Charset.defaultCharset;
-
 import static org.neo4j.kernel.impl.util.Converters.withDefault;
 import static org.neo4j.tooling.ImportTool.csvConfiguration;
 import static org.neo4j.tooling.ImportTool.extractInputFiles;
@@ -100,7 +98,7 @@ class CsvImporter implements Importer
     public void doImport() throws IOException
     {
         FileSystemAbstraction fs = outsideWorld.fileSystem();
-        File storeDir = databaseConfig.get( DatabaseManagementSystemSettings.database_path );
+        File storeDir = databaseConfig.get( GraphDatabaseSettings.database_path );
         File logsDir = databaseConfig.get( GraphDatabaseSettings.logs_directory );
         File reportFile = new File( reportFileName );
 

community/dbms/src/main/java/org/neo4j/commandline/dbms/DatabaseImporter.java

@@ -30,7 +30,7 @@
 import org.neo4j.kernel.impl.util.Converters;
 import org.neo4j.kernel.impl.util.Validators;
 
-import static org.neo4j.dbms.DatabaseManagementSystemSettings.database_path;
+import static org.neo4j.graphdb.factory.GraphDatabaseSettings.database_path;
 
 class DatabaseImporter implements Importer
 {

community/dbms/src/main/java/org/neo4j/commandline/dbms/DumpCommand.java

@@ -32,16 +32,17 @@
 import org.neo4j.commandline.admin.CommandFailed;
 import org.neo4j.commandline.admin.IncorrectUsage;
 import org.neo4j.commandline.arguments.Arguments;
-import org.neo4j.dbms.DatabaseManagementSystemSettings;
 import org.neo4j.dbms.archive.Dumper;
+import org.neo4j.graphdb.factory.GraphDatabaseSettings;
 import org.neo4j.kernel.StoreLockException;
 import org.neo4j.kernel.configuration.Config;
 import org.neo4j.kernel.impl.util.Validators;
 import org.neo4j.kernel.internal.locker.StoreLocker;
 
 import static java.lang.String.format;
 import static org.neo4j.commandline.Util.canonicalPath;
-import static org.neo4j.dbms.DatabaseManagementSystemSettings.database_path;
+import static org.neo4j.graphdb.factory.GraphDatabaseSettings.database_path;
+import static org.neo4j.graphdb.factory.GraphDatabaseSettings.logical_logs_location;
 
 public class DumpCommand implements AdminCommand
 {
@@ -66,7 +67,10 @@ public void execute( String[] args ) throws IncorrectUsage, CommandFailed
     {
         String database = arguments.parse( args ).get( "database" );
         Path archive = calculateArchive( database, arguments.getMandatoryPath( "to" ) );
-        Path databaseDirectory = canonicalPath( toDatabaseDirectory( database ) );
+
+        Config config = buildConfig( database );
+        Path databaseDirectory = canonicalPath( getDatabaseDirectory( config ) );
+        Path transactionLogsDirectory = canonicalPath( getTransactionalLogsDirectory( config ) );
 
         try
         {
@@ -79,7 +83,7 @@ public void execute( String[] args ) throws IncorrectUsage, CommandFailed
 
         try ( Closeable ignored = StoreLockChecker.check( databaseDirectory ) )
         {
-            dump( database, databaseDirectory, archive );
+            dump( database, databaseDirectory, transactionLogsDirectory, archive );
         }
         catch ( StoreLockException e )
         {
@@ -97,25 +101,36 @@ public void execute( String[] args ) throws IncorrectUsage, CommandFailed
         }
     }
 
-    private Path toDatabaseDirectory( String databaseName )
+    private Path getDatabaseDirectory( Config config )
+    {
+        return config.get( database_path ).toPath();
+    }
+
+    private Path getTransactionalLogsDirectory( Config config )
+    {
+        return config.get( logical_logs_location ).toPath();
+    }
+
+    private Config buildConfig( String databaseName )
     {
         return Config.fromFile( configDir.resolve( Config.DEFAULT_CONFIG_FILE_NAME ) )
                 .withHome( homeDir )
                 .withConnectorsDisabled()
-                .withSetting( DatabaseManagementSystemSettings.active_database, databaseName )
-                .build().get( database_path ).toPath();
+                .withSetting( GraphDatabaseSettings.active_database, databaseName )
+                .build();
     }
 
     private Path calculateArchive( String database, Path to )
     {
         return Files.isDirectory( to ) ? to.resolve( database + ".dump" ) : to;
     }
 
-    private void dump( String database, Path databaseDirectory, Path archive ) throws CommandFailed
+    private void dump( String database, Path databaseDirectory, Path transactionalLogsDirectory, Path archive )
+            throws CommandFailed
     {
         try
         {
-            dumper.dump( databaseDirectory, archive, this::isStoreLock );
+            dumper.dump( databaseDirectory, transactionalLogsDirectory, archive, this::isStoreLock );
         }
         catch ( FileAlreadyExistsException e )
         {

community/dbms/src/main/java/org/neo4j/commandline/dbms/ImportCommand.java

@@ -34,7 +34,7 @@
 import org.neo4j.commandline.arguments.OptionalBooleanArg;
 import org.neo4j.commandline.arguments.OptionalNamedArg;
 import org.neo4j.commandline.arguments.OptionalNamedArgWithMetadata;
-import org.neo4j.dbms.DatabaseManagementSystemSettings;
+import org.neo4j.graphdb.factory.GraphDatabaseSettings;
 import org.neo4j.helpers.Args;
 import org.neo4j.helpers.collection.MapUtil;
 import org.neo4j.kernel.configuration.Config;
@@ -250,7 +250,7 @@ public void execute( String[] args ) throws IncorrectUsage, CommandFailed
             Config config =
                     loadNeo4jConfig( homeDir, configDir, database, loadAdditionalConfig( additionalConfigFile ) );
             Validators.CONTAINS_NO_EXISTING_DATABASE
-                    .validate( config.get( DatabaseManagementSystemSettings.database_path ) );
+                    .validate( config.get( GraphDatabaseSettings.database_path ) );
 
             Importer importer = importerFactory.getImporterForMode( mode, Args.parse( args ), config, outsideWorld );
             importer.doImport();
@@ -288,7 +288,7 @@ private static Config loadNeo4jConfig( Path homeDir, Path configDir, String data
     {
         return Config.fromFile( configDir.resolve( Config.DEFAULT_CONFIG_FILE_NAME ) )
                 .withHome( homeDir )
-                .withSetting( DatabaseManagementSystemSettings.active_database, databaseName )
+                .withSetting( GraphDatabaseSettings.active_database, databaseName )
                 .withSettings( additionalConfig )
                 .withConnectorsDisabled().build();
     }