Incremental backups now rotate transaction log files

public/enterprise/backup/src/main/java/org/neo4j/backup/impl/BackupDelegator.java

@@ -55,14 +55,14 @@ class BackupDelegator extends LifecycleAdapter
 
     void copy( AdvertisedSocketAddress fromAddress, StoreId expectedStoreId, Path destDir ) throws StoreCopyFailedException
     {
-        remoteStore.copy( new CatchupAddressProvider.SingleAddressProvider( fromAddress ), expectedStoreId, destDir.toFile() );
+        remoteStore.copy( new CatchupAddressProvider.SingleAddressProvider( fromAddress ), expectedStoreId, destDir.toFile(), true );
     }
 
     CatchupResult tryCatchingUp( AdvertisedSocketAddress fromAddress, StoreId expectedStoreId, Path storeDir ) throws StoreCopyFailedException
     {
         try
         {
-            return remoteStore.tryCatchingUp( fromAddress, expectedStoreId, storeDir.toFile(), true );
+            return remoteStore.tryCatchingUp( fromAddress, expectedStoreId, storeDir.toFile(), true, true );
         }
         catch ( IOException e )
         {

public/enterprise/backup/src/main/java/org/neo4j/backup/impl/OnlineBackupContextFactory.java

@@ -42,9 +42,11 @@
 import org.neo4j.commandline.arguments.common.OptionalCanonicalPath;
 import org.neo4j.consistency.checking.full.ConsistencyFlags;
 import org.neo4j.graphdb.config.Setting;
+import org.neo4j.graphdb.factory.GraphDatabaseSettings;
 import org.neo4j.helpers.TimeUtil;
 import org.neo4j.helpers.collection.MapUtil;
 import org.neo4j.kernel.configuration.Config;
+import org.neo4j.kernel.configuration.Settings;
 import org.neo4j.kernel.impl.util.OptionalHostnamePort;
 
 import static org.neo4j.consistency.ConsistencyCheckSettings.consistency_check_graph;
@@ -188,9 +190,11 @@ public OnlineBackupContext createContext( String... args ) throws IncorrectUsage
                                    .withConnectorsDisabled()
                                    .build();
             additionalConfig.map( this::loadAdditionalConfigFile ).ifPresent( config::augment );
+
             // We only replace the page cache memory setting.
             // Any other custom page swapper, etc. settings are preserved and used.
             config.augment( pagecache_memory, pagecacheMemory );
+            overrideConfigWithBackupSpecificSettings( config );
 
             // Build consistency-checker configuration.
             // Note: We can remove the loading from config file in 4.0.
@@ -213,6 +217,12 @@ public OnlineBackupContext createContext( String... args ) throws IncorrectUsage
         }
     }
 
+    private void overrideConfigWithBackupSpecificSettings( Config config )
+    {
+        // We don't want to pile up tx logs
+        config.augment( GraphDatabaseSettings.logical_log_rotation_threshold, "1m" ); // Forces rotations to be performed when catching up
+    }
+
     private Path getBackupDirectory( Arguments arguments ) throws CommandFailed
     {
         Path path = arguments.getMandatoryPath( ARG_NAME_BACKUP_DIRECTORY );

public/enterprise/backup/src/test/java/org/neo4j/backup/impl/BackupDelegatorTest.java

@@ -77,7 +77,7 @@ public void tryCatchingUpDelegatesToRemoteStore() throws StoreCopyFailedExceptio
         subject.tryCatchingUp( fromAddress, expectedStoreId, storeDir );
 
         // then
-        verify( remoteStore ).tryCatchingUp( fromAddress, expectedStoreId, storeDir.toFile(), true );
+        verify( remoteStore ).tryCatchingUp( fromAddress, expectedStoreId, storeDir.toFile(), true, true );
     }
 
     @Test
@@ -130,7 +130,7 @@ public void retrieveStoreDelegatesToStoreCopyService()
 
         // then
         ArgumentCaptor<CatchupAddressProvider> argumentCaptor = ArgumentCaptor.forClass( CatchupAddressProvider.class );
-        verify( remoteStore ).copy( argumentCaptor.capture(), eq( storeId ), eq( anyFile.toFile() ) );
+        verify( remoteStore ).copy( argumentCaptor.capture(), eq( storeId ), eq( anyFile.toFile() ), eq( true ) );
 
         //and
         assertEquals( anyAddress, argumentCaptor.getValue().primary() );

public/enterprise/backup/src/test/java/org/neo4j/backup/impl/BackupStrategyCoordinatorTest.java

@@ -231,7 +231,8 @@ public void commandFailedWhenConsistencyCheckFails() throws ConsistencyCheckInco
     public void havingNoStrategiesCausesAllSolutionsFailedException() throws CommandFailed
     {
         // given there are no strategies in the solution
-        subject = new BackupStrategyCoordinator( consistencyCheckService, outsideWorld, logProvider, progressMonitorFactory, Collections.emptyList() );
+        subject = new BackupStrategyCoordinator( consistencyCheckService, outsideWorld, logProvider, progressMonitorFactory,
+                Collections.emptyList() );
 
         // then we want a predictable exception (instead of NullPointer)
         expectedException.expect( CommandFailed.class );

public/enterprise/backup/src/test/java/org/neo4j/backup/impl/BackupTransactionLogFilesHelper.java

@@ -0,0 +1,43 @@
+/*
+ * Copyright (c) 2002-2018 "Neo4j,"
+ * Neo4j Sweden AB [http://neo4j.com]
+ *
+ * This file is part of Neo4j Enterprise Edition. The included source
+ * code can be redistributed and/or modified under the terms of the
+ * GNU AFFERO GENERAL PUBLIC LICENSE Version 3
+ * (http://www.fsf.org/licensing/licenses/agpl-3.0.html) with the
+ * Commons Clause, as found in the associated LICENSE.txt file.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * Neo4j object code can be licensed independently from the source
+ * under separate terms from the AGPL. Inquiries can be directed to:
+ * licensing@neo4j.com
+ *
+ * More information is also available at:
+ * https://neo4j.com/licensing/
+ */
+package org.neo4j.backup.impl;
+
+import java.io.File;
+import java.io.IOException;
+
+import org.neo4j.io.fs.DefaultFileSystemAbstraction;
+import org.neo4j.io.fs.FileSystemAbstraction;
+import org.neo4j.io.pagecache.PageCache;
+import org.neo4j.kernel.impl.pagecache.ConfigurableStandalonePageCacheFactory;
+import org.neo4j.kernel.impl.transaction.log.files.LogFiles;
+import org.neo4j.kernel.impl.transaction.log.files.LogFilesBuilder;
+
+public class BackupTransactionLogFilesHelper
+{
+    LogFiles readLogFiles( File backupDir ) throws IOException
+    {
+        FileSystemAbstraction fileSystemAbstraction = new DefaultFileSystemAbstraction();
+        PageCache pageCache = ConfigurableStandalonePageCacheFactory.createPageCache( fileSystemAbstraction );
+        return LogFilesBuilder.activeFilesBuilder( backupDir, fileSystemAbstraction, pageCache ).build();
+    }
+}

public/enterprise/backup/src/test/java/org/neo4j/backup/impl/OnlineBackupCommandCcIT.java

@@ -56,6 +56,7 @@
 import org.neo4j.kernel.impl.enterprise.configuration.OnlineBackupSettings;
 import org.neo4j.kernel.impl.store.format.highlimit.HighLimit;
 import org.neo4j.kernel.impl.store.format.standard.Standard;
+import org.neo4j.kernel.impl.transaction.log.files.LogFiles;
 import org.neo4j.ports.allocation.PortAuthority;
 import org.neo4j.test.DbRepresentation;
 import org.neo4j.test.causalclustering.ClusterRule;
@@ -68,6 +69,7 @@
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertTrue;
 import static org.junit.Assume.assumeFalse;
+import static org.neo4j.backup.impl.OnlineBackupCommandHaIT.transactions1M;
 
 @RunWith( Parameterized.class )
 public class OnlineBackupCommandCcIT
@@ -269,6 +271,44 @@ public void reportsProgress() throws Exception
         assertTrue( output.contains( "Finished receiving index snapshots" ) );
     }
 
+    @Test
+    public void onlyTheLatestTransactionIsKeptAfterIncrementalBackup() throws Exception
+    {
+        // given database exists with data
+        Cluster cluster = startCluster( recordFormat );
+        createSomeData( cluster );
+
+        // and we have a full backup
+        String backupName = "backupName" + recordFormat;
+        String address = CausalClusteringTestHelpers.backupAddress( clusterLeader( cluster ).database() );
+        assertEquals( 0, runBackupToolFromOtherJvmToGetExitCode(
+                "--from", address,
+                "--cc-report-dir=" + backupDir,
+                "--backup-dir=" + backupDir,
+                "--name=" + backupName ) );
+
+        // and the database contains a few more transactions
+        transactions1M( clusterLeader( cluster ).database() ); // first rotation
+        transactions1M( clusterLeader( cluster ).database() ); // second rotation and prune
+
+        // when we perform an incremental backup
+        assertEquals( 0, runBackupToolFromSameJvm(
+                "--from", address,
+                "--cc-report-dir=" + backupDir,
+                "--backup-dir=" + backupDir,
+                "--name=" + backupName ) );
+
+        // then there has been a rotation
+        BackupTransactionLogFilesHelper backupTransactionLogFilesHelper = new BackupTransactionLogFilesHelper();
+        LogFiles logFiles = backupTransactionLogFilesHelper.readLogFiles( backupDir.toPath().resolve( backupName ).toFile() );
+        long highestTxIdInLogFiles = logFiles.getHighestLogVersion();
+        assertEquals( 2, highestTxIdInLogFiles );
+
+        // and the original log has not been removed since the transactions are applied at start
+        long lowestTxIdInLogFiles = logFiles.getLowestLogVersion();
+        assertEquals( 0, lowestTxIdInLogFiles );
+    }
+
     static PrintStream wrapWithNormalOutput( PrintStream normalOutput, PrintStream nullAbleOutput )
     {
         if ( nullAbleOutput == null )
@@ -377,12 +417,17 @@ private int runBackupToolFromOtherJvmToGetExitCode( String... args ) throws Exce
         return TestHelpers.runBackupToolFromOtherJvmToGetExitCode( testDirectory.absolutePath(), args );
     }
 
+    private int runBackupToolFromSameJvm( String... args ) throws Exception
+    {
+        return runBackupToolFromSameJvmToGetExitCode( testDirectory.absolutePath(), testDirectory.absolutePath().getName(), args );
+    }
+
     /**
      * This unused method is used for debugging, so don't remove
      */
-    private int runBackupToolFromSameJvmToGetExitCode( String... args ) throws Exception
+    public static int runBackupToolFromSameJvmToGetExitCode( File backupDir, String backupName, String... args ) throws Exception
     {
-        return new OnlineBackupCommandBuilder().withRawArgs( args ).backup( testDirectory.absolutePath(), testDirectory.absolutePath().getName() )
+        return new OnlineBackupCommandBuilder().withRawArgs( args ).backup( backupDir, backupName )
                ? 0 : 1;
     }
 }

public/enterprise/backup/src/test/java/org/neo4j/backup/impl/OnlineBackupCommandHaIT.java

@@ -41,6 +41,7 @@
 import java.util.Arrays;
 import java.util.List;
 import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.stream.LongStream;
 
 import org.neo4j.graphdb.DatabaseShutdownException;
 import org.neo4j.graphdb.GraphDatabaseService;
@@ -49,6 +50,7 @@
 import org.neo4j.graphdb.RelationshipType;
 import org.neo4j.graphdb.Transaction;
 import org.neo4j.graphdb.factory.GraphDatabaseSettings;
+import org.neo4j.io.ByteUnit;
 import org.neo4j.io.fs.DefaultFileSystemAbstraction;
 import org.neo4j.kernel.configuration.Config;
 import org.neo4j.kernel.configuration.Settings;
@@ -60,9 +62,11 @@
 import org.neo4j.ports.allocation.PortAuthority;
 import org.neo4j.test.DbRepresentation;
 import org.neo4j.test.rule.EmbeddedDatabaseRule;
+import org.neo4j.test.rule.SuppressOutput;
 import org.neo4j.test.rule.TestDirectory;
 
 import static java.lang.String.format;
+import static java.util.stream.Collectors.joining;
 import static org.hamcrest.Matchers.greaterThan;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
@@ -81,7 +85,10 @@ public class OnlineBackupCommandHaIT
     private final EmbeddedDatabaseRule db = new EmbeddedDatabaseRule().startLazily();
 
     @Rule
-    public final RuleChain ruleChain = RuleChain.outerRule( testDirectory ).around( db );
+    public final SuppressOutput suppressOutput = SuppressOutput.suppressAll();
+
+    @Rule
+    public final RuleChain ruleChain = RuleChain.outerRule( suppressOutput ).around( testDirectory ).around( db );
 
     private File backupDir;
 
@@ -278,6 +285,64 @@ public void reportsProgress() throws Exception
         assertFalse( output.contains( "Finished receiving index snapshots" ) );
     }
 
+    @Test
+    public void onlyTheLatestTransactionIsKeptAfterIncrementalBackup() throws Exception
+    {
+        // given database exists with data
+        int port = PortAuthority.allocatePort();
+        startDb( port );
+        createSomeData( db );
+
+        // and we have a full backup
+        String backupName = "backupName" + recordFormat;
+        String address = "localhost:" + port;
+        assertEquals( 0, runBackupToolFromOtherJvmToGetExitCode( backupDir,
+                "--from", address,
+                "--cc-report-dir=" + backupDir,
+                "--backup-dir=" + backupDir,
+                "--protocol=common",
+                "--name=" + backupName ) );
+
+        // and the database contains a few more transactions
+        transactions1M( db ); // first rotation
+        transactions1M( db ); // second rotation, prune happens at recovery but only in common protocol
+
+        // when we perform an incremental backup
+        assertEquals( 0, runBackupToolFromOtherJvmToGetExitCode( backupDir,
+                "--from", address,
+                "--cc-report-dir=" + backupDir,
+                "--backup-dir=" + backupDir,
+                "--protocol=common",
+                "--name=" + backupName ) );
+
+        // then there has been a rotation
+        BackupTransactionLogFilesHelper backupTransactionLogFilesHelper = new BackupTransactionLogFilesHelper();
+        LogFiles logFiles = backupTransactionLogFilesHelper.readLogFiles( backupDir.toPath().resolve( backupName ).toFile() );
+        long highestTxIdInLogFiles = logFiles.getHighestLogVersion();
+        assertEquals( 2, highestTxIdInLogFiles );
+
+        // and the original log has not been removed since the transactions are applied at start
+        long lowestTxIdInLogFiles = logFiles.getLowestLogVersion();
+        assertEquals( 0, lowestTxIdInLogFiles );
+    }
+
+    static void transactions1M( GraphDatabaseService db )
+    {
+        int numberOfTransactions = 500;
+        long sizeOfTransaction = (ByteUnit.mebiBytes( 1 ) / numberOfTransactions) + 1;
+        for ( int txId = 0; txId < numberOfTransactions; txId++ )
+        {
+            try ( Transaction tx = db.beginTx() )
+            {
+                Node node = db.createNode();
+                String longString = LongStream.range( 0, sizeOfTransaction ).map( l -> l % 10 ).mapToObj( Long::toString ).collect( joining( "" ) );
+                node.setProperty( "name", longString );
+                db.createNode().createRelationshipTo( node, RelationshipType.withName( "KNOWS" ) );
+                tx.success();
+            }
+        }
+    }
+
     private static void createSomeData( GraphDatabaseService db )
     {
         try ( Transaction tx = db.beginTx() )

public/enterprise/backup/src/test/java/org/neo4j/backup/impl/OnlineBackupContextFactoryTest.java

@@ -39,6 +39,8 @@
 import org.neo4j.causalclustering.core.CausalClusteringSettings;
 import org.neo4j.commandline.admin.CommandFailed;
 import org.neo4j.commandline.admin.IncorrectUsage;
+import org.neo4j.graphdb.factory.GraphDatabaseSettings;
+import org.neo4j.io.ByteUnit;
 import org.neo4j.kernel.configuration.Config;
 import org.neo4j.test.rule.SuppressOutput;
 import org.neo4j.test.rule.TestDirectory;
@@ -331,6 +333,18 @@ public void overrideWithLegacy() throws CommandFailed, IncorrectUsage
         }
     }
 
+    @Test
+    public void configIsProvided1mRotation() throws CommandFailed, IncorrectUsage
+    {
+        OnlineBackupContextFactory builder = new OnlineBackupContextFactory( homeDir, configDir );
+
+        //
+        OnlineBackupContext context = builder.createContext( requiredAnd() );
+
+        // then
+        assertEquals( ByteUnit.mebiBytes( 1 ), context.getConfig().get( GraphDatabaseSettings.logical_log_rotation_threshold ).longValue() );
+    }
+
     private String[] requiredAnd( String... additionalArgs )
     {
         List<String> args = new ArrayList<>();

public/enterprise/causal-clustering/src/main/java/org/neo4j/causalclustering/catchup/storecopy/CopiedStoreRecovery.java

@@ -104,7 +104,7 @@ private GraphDatabaseService newTempDatabase( File tempStore )
                 .newEmbeddedDatabaseBuilder( tempStore )
                 .setConfig( OnlineBackupSettings.online_backup_enabled, Settings.FALSE )
                 .setConfig( GraphDatabaseSettings.pagecache_warmup_enabled, Settings.FALSE )
-                .setConfig( GraphDatabaseSettings.keep_logical_logs, Settings.TRUE )
+                .setConfig( GraphDatabaseSettings.keep_logical_logs, Settings.FALSE )
                 .setConfig( GraphDatabaseSettings.allow_upgrade,
                         config.get( GraphDatabaseSettings.allow_upgrade ).toString() )
                 .setConfig( GraphDatabaseSettings.record_format, config.get( GraphDatabaseSettings.record_format ) )