From 22f3c8369ced48e10f913d81ad697dd472d0cfdd Mon Sep 17 00:00:00 2001 From: MishaDemianenko Date: Fri, 24 Nov 2017 00:19:03 +0100 Subject: [PATCH] Online backup transaction logs location Put transaction logs into the same folder as backup directory always. Previously online backup tried to put them into a folder that was related to neo4j_home that can be either empty either incorrect for backup case. --- .../OnlineBackupCommandConfigLoader.java | 5 +++- .../org/neo4j/backup/OnlineBackupContext.java | 2 +- .../backup/OnlineBackupContextLoader.java | 5 ++-- .../backup/OnlineBackupRequiredArguments.java | 6 ++++ .../OnlineBackupCommandConfigLoaderTest.java | 7 +++-- .../neo4j/backup/OnlineBackupCommandHaIT.java | 28 +++++++++++++++++-- 6 files changed, 44 insertions(+), 9 deletions(-) diff --git a/enterprise/backup/src/main/java/org/neo4j/backup/OnlineBackupCommandConfigLoader.java b/enterprise/backup/src/main/java/org/neo4j/backup/OnlineBackupCommandConfigLoader.java index a1b80478f06e..7f35c5122147 100644 --- a/enterprise/backup/src/main/java/org/neo4j/backup/OnlineBackupCommandConfigLoader.java +++ b/enterprise/backup/src/main/java/org/neo4j/backup/OnlineBackupCommandConfigLoader.java @@ -19,11 +19,13 @@ */ package org.neo4j.backup; +import java.io.File; import java.io.IOException; import java.nio.file.Path; import java.util.Optional; import org.neo4j.commandline.admin.CommandFailed; +import org.neo4j.graphdb.factory.GraphDatabaseSettings; import org.neo4j.helpers.collection.MapUtil; import org.neo4j.kernel.configuration.Config; @@ -38,9 +40,10 @@ class OnlineBackupCommandConfigLoader this.configDir = configDir; } - Config loadConfig( Optional additionalConfig ) throws CommandFailed + Config loadConfig( Optional additionalConfig, File folder ) throws CommandFailed { Config config = Config.fromFile( configDir.resolve( Config.DEFAULT_CONFIG_FILE_NAME ) ).withHome( homeDir ) + .withSetting( GraphDatabaseSettings.logical_logs_location, folder.getAbsolutePath() ) .withConnectorsDisabled().build(); return withAdditionalConfig( additionalConfig, config ); } diff --git a/enterprise/backup/src/main/java/org/neo4j/backup/OnlineBackupContext.java b/enterprise/backup/src/main/java/org/neo4j/backup/OnlineBackupContext.java index dab1d84e41f0..6acae9487cd3 100644 --- a/enterprise/backup/src/main/java/org/neo4j/backup/OnlineBackupContext.java +++ b/enterprise/backup/src/main/java/org/neo4j/backup/OnlineBackupContext.java @@ -54,6 +54,6 @@ public ConsistencyFlags getConsistencyFlags() public File getResolvedLocationFromName() { - return requiredArguments.getFolder().resolve( requiredArguments.getName() ).toFile(); + return requiredArguments.getResolvedLocationFromName(); } } diff --git a/enterprise/backup/src/main/java/org/neo4j/backup/OnlineBackupContextLoader.java b/enterprise/backup/src/main/java/org/neo4j/backup/OnlineBackupContextLoader.java index 7113632e88a9..ef9fa67bd811 100644 --- a/enterprise/backup/src/main/java/org/neo4j/backup/OnlineBackupContextLoader.java +++ b/enterprise/backup/src/main/java/org/neo4j/backup/OnlineBackupContextLoader.java @@ -32,7 +32,7 @@ public class OnlineBackupContextLoader private final BackupCommandArgumentHandler backupCommandArgumentHandler; private final OnlineBackupCommandConfigLoader onlineBackupCommandConfigLoader; - public OnlineBackupContextLoader( BackupCommandArgumentHandler backupCommandArgumentHandler, + OnlineBackupContextLoader( BackupCommandArgumentHandler backupCommandArgumentHandler, OnlineBackupCommandConfigLoader onlineBackupCommandConfigLoader ) { this.backupCommandArgumentHandler = backupCommandArgumentHandler; @@ -49,7 +49,8 @@ public OnlineBackupContext fromCommandLineArguments( String[] commandlineArgs ) try { OnlineBackupRequiredArguments requiredArguments = backupCommandArgumentHandler.establishRequiredArguments( commandlineArgs ); - Config config = onlineBackupCommandConfigLoader.loadConfig( requiredArguments.getAdditionalConfig() ); + Config config = onlineBackupCommandConfigLoader.loadConfig( requiredArguments.getAdditionalConfig(), + requiredArguments.getResolvedLocationFromName() ); ConsistencyFlags consistencyFlags = backupCommandArgumentHandler.readFlagsFromArgumentsOrDefaultToConfig( config ); return new OnlineBackupContext( requiredArguments, config, consistencyFlags ); diff --git a/enterprise/backup/src/main/java/org/neo4j/backup/OnlineBackupRequiredArguments.java b/enterprise/backup/src/main/java/org/neo4j/backup/OnlineBackupRequiredArguments.java index 6c0d287fa0ca..1f32f5f4ee93 100644 --- a/enterprise/backup/src/main/java/org/neo4j/backup/OnlineBackupRequiredArguments.java +++ b/enterprise/backup/src/main/java/org/neo4j/backup/OnlineBackupRequiredArguments.java @@ -19,6 +19,7 @@ */ package org.neo4j.backup; +import java.io.File; import java.nio.file.Path; import java.util.Optional; @@ -87,4 +88,9 @@ public Path getReportDir() { return reportDir; } + + public File getResolvedLocationFromName() + { + return folder.resolve( name ).toFile(); + } } diff --git a/enterprise/backup/src/test/java/org/neo4j/backup/OnlineBackupCommandConfigLoaderTest.java b/enterprise/backup/src/test/java/org/neo4j/backup/OnlineBackupCommandConfigLoaderTest.java index 62920e2d5703..8d16c956d157 100644 --- a/enterprise/backup/src/test/java/org/neo4j/backup/OnlineBackupCommandConfigLoaderTest.java +++ b/enterprise/backup/src/test/java/org/neo4j/backup/OnlineBackupCommandConfigLoaderTest.java @@ -85,7 +85,7 @@ public void errorHandledForNonExistingAdditionalConfigFile() throws CommandFaile // expect assertFalse( additionalConf.exists() ); - subject.loadConfig( Optional.of( additionalConf.toPath() ) ); + subject.loadConfig( Optional.of( additionalConf.toPath() ), new File( "destination" ) ); } @Test @@ -101,7 +101,7 @@ public void prioritiseConfigDirOverHomeDir() throws IOException, CommandFailed appendToFile( homeDirConfigFile, "causal_clustering.raft_in_queue_max_batch=21" ); // when - Config config = subject.loadConfig( Optional.empty() ); + Config config = subject.loadConfig( Optional.empty(), new File( "destination" ) ); // then assertEquals( Integer.valueOf( 4 ), config.get( CausalClusteringSettings.expected_core_cluster_size ) ); @@ -121,7 +121,8 @@ public void prioritiseAdditionalOverConfigDir() throws IOException, CommandFaile appendToFile( additionalConfigFile, "causal_clustering.expected_core_cluster_size=5" ); // when - Config config = subject.loadConfig( Optional.of( additionalConfigFile.toPath() ) ); + Config config = subject.loadConfig( Optional.of( additionalConfigFile.toPath() ), + new File( "destination" ) ); // then assertEquals( Integer.valueOf( 5 ), config.get( CausalClusteringSettings.expected_core_cluster_size ) ); diff --git a/enterprise/backup/src/test/java/org/neo4j/backup/OnlineBackupCommandHaIT.java b/enterprise/backup/src/test/java/org/neo4j/backup/OnlineBackupCommandHaIT.java index 773c7f053dd0..d05147d92801 100644 --- a/enterprise/backup/src/test/java/org/neo4j/backup/OnlineBackupCommandHaIT.java +++ b/enterprise/backup/src/test/java/org/neo4j/backup/OnlineBackupCommandHaIT.java @@ -44,18 +44,24 @@ import org.neo4j.graphdb.RelationshipType; import org.neo4j.graphdb.Transaction; import org.neo4j.graphdb.factory.GraphDatabaseSettings; +import org.neo4j.io.fs.DefaultFileSystemAbstraction; import org.neo4j.kernel.configuration.Config; import org.neo4j.kernel.configuration.Settings; import org.neo4j.kernel.impl.enterprise.configuration.OnlineBackupSettings; import org.neo4j.kernel.impl.store.format.highlimit.HighLimit; import org.neo4j.kernel.impl.store.format.standard.Standard; +import org.neo4j.kernel.impl.transaction.log.files.LogFiles; +import org.neo4j.kernel.impl.transaction.log.files.LogFilesBuilder; import org.neo4j.ports.allocation.PortAuthority; import org.neo4j.test.DbRepresentation; import org.neo4j.test.rule.EmbeddedDatabaseRule; import org.neo4j.test.rule.SuppressOutput; import org.neo4j.test.rule.TestDirectory; +import static org.hamcrest.Matchers.greaterThan; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertTrue; import static org.junit.Assume.assumeFalse; import static org.neo4j.util.TestHelpers.runBackupToolFromOtherJvmToGetExitCode; @@ -86,7 +92,7 @@ public static List recordFormats() private static final String PROP_NAME = "name"; private static final String PROP_RANDOM = "random"; - private static DbRepresentation createSomeData( GraphDatabaseService db ) + private static void createSomeData( GraphDatabaseService db ) { try ( Transaction tx = db.beginTx() ) { @@ -95,7 +101,6 @@ private static DbRepresentation createSomeData( GraphDatabaseService db ) db.createNode().createRelationshipTo( node, RelationshipType.withName( "KNOWS" ) ); tx.success(); } - return DbRepresentation.of( db ); } @Before @@ -162,6 +167,25 @@ public void dataIsInAUsableStateAfterBackup() throws Exception db.shutdown(); } + @Test + public void backupDatabaseTransactionLogsStoredWithDatabase() throws Exception + { + int backupPort = PortAuthority.allocatePort(); + startDb( backupPort ); + String ip = ":" + backupPort; + String name = "backupWithTxLogs" + recordFormat; + assertEquals( 0, runBackupTool( "--from", ip, "--cc-report-dir=" + backupDir, "--backup-dir=" + backupDir, + "--name=" + name ) ); + db.shutdown(); + + try ( DefaultFileSystemAbstraction fileSystem = new DefaultFileSystemAbstraction() ) + { + LogFiles logFiles = LogFilesBuilder.logFilesBasedOnlyBuilder( new File( backupDir, name ), fileSystem ).build(); + assertTrue( logFiles.versionExists( 0 ) ); + assertThat( logFiles.getLogFileForVersion( 0 ).length(), greaterThan( 50L ) ); + } + } + private void repeatedlyPopulateDatabase( GraphDatabaseService db, AtomicBoolean continueFlagReference ) { while ( continueFlagReference.get() )