Improve ClusterSeedingIT to cover more scenarios

* Split up ClusterSeedingIT in two parameterized tests.
ClusterSeedingIT and SeedingNewMemberIt.
* Each test will seed with each type of BackupStore. EmptyBackup,
BackupContainingSomeData, BackupContainingSomeDataAndNoTransactionLogs
* Added monitors to be able to track and assert on store copy.

enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/backup/BackupCoreIT.java

@@ -24,28 +24,20 @@
 import org.junit.Test;
 
 import java.io.File;
-import java.net.InetSocketAddress;
-import java.util.ArrayList;
-import java.util.List;
 
 import org.neo4j.backup.OnlineBackupSettings;
-import org.neo4j.causalclustering.core.CausalClusteringSettings;
-import org.neo4j.causalclustering.core.CoreGraphDatabase;
 import org.neo4j.causalclustering.discovery.Cluster;
 import org.neo4j.causalclustering.discovery.CoreClusterMember;
-import org.neo4j.graphdb.Node;
-import org.neo4j.graphdb.factory.GraphDatabaseSettings;
-import org.neo4j.helpers.collection.MapUtil;
-import org.neo4j.kernel.configuration.Config;
-import org.neo4j.kernel.impl.factory.GraphDatabaseFacade;
-import org.neo4j.kernel.impl.store.format.standard.Standard;
 import org.neo4j.test.DbRepresentation;
 import org.neo4j.test.causalclustering.ClusterRule;
 
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertNotEquals;
 import static org.neo4j.backup.OnlineBackupCommandIT.runBackupToolFromOtherJvmToGetExitCode;
-import static org.neo4j.graphdb.Label.label;
+import static org.neo4j.causalclustering.helpers.BackupUtil.backupAddress;
+import static org.neo4j.causalclustering.helpers.BackupUtil.backupArguments;
+import static org.neo4j.causalclustering.helpers.BackupUtil.getConfig;
+import static org.neo4j.causalclustering.helpers.DataCreator.createSomeData;
 
 public class BackupCoreIT
 {
@@ -86,37 +78,4 @@ public void makeSureBackupCanBePerformedFromAnyInstance() throws Throwable
             assertNotEquals( backupRepresentation, afterChange );
         }
     }
-
-    static CoreGraphDatabase createSomeData( Cluster cluster ) throws Exception
-    {
-        return cluster.coreTx( ( db, tx ) ->
-        {
-            Node node = db.createNode( label( "boo" ) );
-            node.setProperty( "foobar", "baz_bat" );
-            tx.success();
-        } ).database();
-    }
-
-    static String backupAddress( GraphDatabaseFacade db )
-    {
-        InetSocketAddress inetSocketAddress = db.getDependencyResolver()
-                .resolveDependency( Config.class ).get( CausalClusteringSettings.transaction_advertised_address )
-                .socketAddress();
-        return inetSocketAddress.getHostName() + ":" + (inetSocketAddress.getPort() + 2000);
-    }
-
-    static String[] backupArguments( String from, File backupsDir, String name )
-    {
-        List<String> args = new ArrayList<>();
-        args.add( "--from=" + from );
-        args.add( "--cc-report-dir=" + backupsDir );
-        args.add( "--backup-dir=" + backupsDir );
-        args.add( "--name=" + name );
-        return args.toArray( new String[args.size()] );
-    }
-
-    static Config getConfig()
-    {
-        return Config.embeddedDefaults( MapUtil.stringMap( GraphDatabaseSettings.record_format.name(), Standard.LATEST_NAME ) );
-    }
 }

enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/backup/BackupReadReplicaIT.java

@@ -43,14 +43,14 @@
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertNotEquals;
 import static org.neo4j.backup.OnlineBackupCommandIT.runBackupToolFromOtherJvmToGetExitCode;
-import static org.neo4j.causalclustering.backup.BackupCoreIT.backupArguments;
-import static org.neo4j.causalclustering.backup.BackupCoreIT.createSomeData;
-import static org.neo4j.causalclustering.backup.BackupCoreIT.getConfig;
+import static org.neo4j.causalclustering.helpers.BackupUtil.backupArguments;
+import static org.neo4j.causalclustering.helpers.BackupUtil.getConfig;
+import static org.neo4j.causalclustering.helpers.DataCreator.createSomeData;
 import static org.neo4j.function.Predicates.awaitEx;
 
 public class BackupReadReplicaIT
 {
-    private int backupPort = findFreePort( 22000, 23000);
+    private int backupPort = findFreePort( 22000, 23000 );
 
     @Rule
     public ClusterRule clusterRule = new ClusterRule( BackupReadReplicaIT.class ).withNumberOfCoreMembers( 3 )

enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/backup/ClusterSeedingIT.java

@@ -21,16 +21,21 @@
 
 import org.junit.After;
 import org.junit.Before;
-import org.junit.Ignore;
 import org.junit.Rule;
 import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
 
 import java.io.File;
+import java.util.Arrays;
 import java.util.Map;
 import java.util.function.IntFunction;
 
 import org.neo4j.backup.OnlineBackupSettings;
-import org.neo4j.causalclustering.core.CoreGraphDatabase;
+import org.neo4j.causalclustering.backup.backup_stores.BackupStore;
+import org.neo4j.causalclustering.backup.backup_stores.BackupStoreWithSomeData;
+import org.neo4j.causalclustering.backup.backup_stores.BackupStoreWithSomeDataButNoTransactionLogs;
+import org.neo4j.causalclustering.backup.backup_stores.EmptyBackupStore;
 import org.neo4j.causalclustering.discovery.Cluster;
 import org.neo4j.causalclustering.discovery.CoreClusterMember;
 import org.neo4j.causalclustering.discovery.IpFamily;
@@ -43,17 +48,34 @@
 
 import static java.util.Collections.emptyMap;
 import static java.util.Collections.singletonMap;
-import static org.junit.Assert.assertEquals;
-import static org.neo4j.backup.OnlineBackupCommandIT.runBackupToolFromOtherJvmToGetExitCode;
-import static org.neo4j.causalclustering.backup.BackupCoreIT.backupAddress;
+import static org.junit.Assert.assertFalse;
 import static org.neo4j.causalclustering.discovery.Cluster.dataMatchesEventually;
-import static org.neo4j.causalclustering.helpers.DataCreator.createEmptyNodes;
+import static org.neo4j.causalclustering.helpers.BackupUtil.restoreFromBackup;
 
+@RunWith( Parameterized.class )
 public class ClusterSeedingIT
 {
     private Cluster backupCluster;
     private Cluster cluster;
     private FileSystemAbstraction fsa;
+    private FileCopyDetector fileCopyDetector;
+
+    @Parameterized.Parameters( name = "{0}" )
+    public static Iterable<BackupStore> data() throws Exception
+    {
+        return stores();
+    }
+
+    private static Iterable<BackupStore> stores()
+    {
+        return Arrays.asList(
+                new EmptyBackupStore(),
+                new BackupStoreWithSomeData(),
+                new BackupStoreWithSomeDataButNoTransactionLogs() );
+    }
+
+    @Parameterized.Parameter()
+    public BackupStore initialStore;
 
     @Rule
     public TestDirectory testDir = TestDirectory.testDirectory();
@@ -63,6 +85,7 @@ public class ClusterSeedingIT
     @Before
     public void setup() throws Exception
     {
+        this.fileCopyDetector = new FileCopyDetector();
         fsa = fileSystemRule.get();
         backupCluster = new Cluster( testDir.directory( "cluster-for-backup" ), 3, 0,
                 new SharedDiscoveryService(), emptyMap(), backupParams(), emptyMap(), emptyMap(), Standard
@@ -95,101 +118,27 @@ public void after() throws Exception
         }
     }
 
-    private File createBackupUsingAnotherCluster() throws Exception
+    @Test
+    public void shouldSeedNewCluster() throws Exception
     {
+        // given
         backupCluster.start();
-        CoreGraphDatabase db = BackupCoreIT.createSomeData( backupCluster );
-
-        File backup = createBackup( db, "some-backup" );
+        File backup = initialStore.get( baseBackupDir, backupCluster );
         backupCluster.shutdown();
 
-        return backup;
-    }
-
-    private File createBackup( CoreGraphDatabase db, String backupName ) throws Exception
-    {
-        String[] args = BackupCoreIT.backupArguments( backupAddress( db ), baseBackupDir, backupName );
-        assertEquals( 0, runBackupToolFromOtherJvmToGetExitCode( testDir.absolutePath(), args ) );
-        return new File( baseBackupDir, backupName );
-    }
+        for ( CoreClusterMember coreClusterMember : cluster.coreMembers() )
+        {
+            restoreFromBackup( backup, fsa, coreClusterMember );
+        }
 
-    @Test
-    public void shouldRestoreBySeedingAllMembers() throws Throwable
-    {
-        // given
-        File backupDir = createBackupUsingAnotherCluster();
-        DbRepresentation before = DbRepresentation.of( backupDir );
+        // we want the cluster to seed from backup. No instance should delete and re-copy the store.
+        cluster.coreMembers().forEach( ccm -> ccm.monitors().addMonitorListener( fileCopyDetector ) );
 
         // when
-        fsa.copyRecursively( backupDir, cluster.getCoreMemberById( 0 ).storeDir() );
-        fsa.copyRecursively( backupDir, cluster.getCoreMemberById( 1 ).storeDir() );
-        fsa.copyRecursively( backupDir, cluster.getCoreMemberById( 2 ).storeDir() );
-
         cluster.start();
 
-        // then
-        dataMatchesEventually( before, cluster.coreMembers() );
-    }
-
-    @Test
-    public void shouldSeedNewMemberFromEmptyIdleCluster() throws Throwable
-    {
-        // given
-        cluster = new Cluster( testDir.directory( "cluster-b" ), 3, 0,
-                new SharedDiscoveryService(), emptyMap(), backupParams(), emptyMap(), emptyMap(), Standard
-                .LATEST_NAME, IpFamily.IPV4, false );
-        cluster.start();
-
-        // when: creating a backup
-        File backupDir = createBackup( cluster.getCoreMemberById( 0 ).database(), "the-backup" );
-
-        // and: seeding new member with said backup
-        CoreClusterMember newMember = cluster.addCoreMemberWithId( 3 );
-        fsa.copyRecursively( backupDir, newMember.storeDir() );
-        newMember.start();
-
-        // then
-        dataMatchesEventually( DbRepresentation.of( newMember.database() ), cluster.coreMembers() );
-    }
-
-    @Test
-    public void shouldSeedNewMemberFromNonEmptyIdleCluster() throws Throwable
-    {
-        // given
-        cluster = new Cluster( testDir.directory( "cluster-b" ), 3, 0,
-                new SharedDiscoveryService(), emptyMap(), backupParams(), emptyMap(), emptyMap(), Standard
-                .LATEST_NAME, IpFamily.IPV4, false );
-        cluster.start();
-        createEmptyNodes( cluster, 100 );
-
-        // when: creating a backup
-        File backupDir = createBackup( cluster.getCoreMemberById( 0 ).database(), "the-backup" );
-
-        // and: seeding new member with said backup
-        CoreClusterMember newMember = cluster.addCoreMemberWithId( 3 );
-        fsa.copyRecursively( backupDir, newMember.storeDir() );
-        newMember.start();
-
-        // then
-        dataMatchesEventually( DbRepresentation.of( newMember.database() ), cluster.coreMembers() );
-    }
-
-    @Test
-    @Ignore( "need to seed all members for now" )
-    public void shouldRestoreBySeedingSingleMember() throws Throwable
-    {
-        // given
-        File backupDir = createBackupUsingAnotherCluster();
-        DbRepresentation before = DbRepresentation.of( backupDir );
-
-        // when
-        fsa.copyRecursively( backupDir, cluster.getCoreMemberById( 0 ).storeDir() );
-        cluster.getCoreMemberById( 0 ).start();
-        Thread.sleep( 2_000 );
-        cluster.getCoreMemberById( 1 ).start();
-        cluster.getCoreMemberById( 2 ).start();
-
-        // then
-        dataMatchesEventually( before, cluster.coreMembers() );
+        //then
+        dataMatchesEventually( DbRepresentation.of( backup ), cluster.coreMembers() );
+        assertFalse( fileCopyDetector.hasDetectedAnyFileCopied() );
     }
 }

enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/backup/FileCopyDetector.java

@@ -0,0 +1,40 @@
+/*
+ * Copyright (c) 2002-2017 "Neo Technology,"
+ * Network Engine for Objects in Lund AB [http://neotechnology.com]
+ *
+ * This file is part of Neo4j.
+ *
+ * Neo4j is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+package org.neo4j.causalclustering.backup;
+
+import java.io.File;
+
+import org.neo4j.causalclustering.catchup.tx.FileCopyMonitor;
+
+public class FileCopyDetector implements FileCopyMonitor
+{
+    private volatile boolean hasCopiedFile = false;
+
+    @Override
+    public void copyFile( File file )
+    {
+        hasCopiedFile = true;
+    }
+
+    public boolean hasDetectedAnyFileCopied()
+    {
+        return hasCopiedFile;
+    }
+}