Add CC step after stopping an istance in stress tests

neo4j · Dec 8, 2016 · e9a6a2c · e9a6a2c
1 parent 16a782b
commit e9a6a2c
Show file tree

Hide file tree

Showing 4 changed files with 66 additions and 6 deletions.
diff --git a/stresstests/pom.xml b/stresstests/pom.xml
@@ -103,6 +103,18 @@
       <scope>test</scope>
       <type>test-jar</type>
     </dependency>
+    <dependency>
+      <groupId>org.neo4j</groupId>
+      <artifactId>neo4j-lucene-index</artifactId>
+      <version>${project.version}</version>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.neo4j</groupId>
+      <artifactId>neo4j-consistency-check</artifactId>
+      <version>${project.version}</version>
+      <scope>test</scope>
+    </dependency>
     <dependency>
       <groupId>org.neo4j</groupId>
       <artifactId>neo4j-causal-clustering</artifactId>

diff --git a/.../java/org/neo4j/causalclustering/stresstests/BackupStoreCopyInteractionStressTesting.java b/.../java/org/neo4j/causalclustering/stresstests/BackupStoreCopyInteractionStressTesting.java
@@ -36,6 +36,8 @@
 import org.neo4j.causalclustering.discovery.HazelcastDiscoveryServiceFactory;
 import org.neo4j.helpers.AdvertisedSocketAddress;
 import org.neo4j.helpers.SocketAddress;
+import org.neo4j.io.fs.DefaultFileSystemAbstraction;
+import org.neo4j.io.fs.FileSystemAbstraction;
 import org.neo4j.io.fs.FileUtils;
 import org.neo4j.kernel.impl.store.format.standard.StandardV3_0;
 
@@ -65,6 +67,8 @@ public class BackupStoreCopyInteractionStressTesting
     private static final String DEFAULT_BASE_CORE_BACKUP_PORT = "8000";
     private static final String DEFAULT_BASE_EDGE_BACKUP_PORT = "9000";
 
+    private final FileSystemAbstraction fs = new DefaultFileSystemAbstraction();
+
     @Test
     public void shouldBehaveCorrectlyUnderStress() throws Exception
     {
@@ -119,8 +123,8 @@ public void shouldBehaveCorrectlyUnderStress() throws Exception
             }
 
             Future<Throwable> workload = service.submit( new Workload( keepGoing, onFailure, cluster ) );
-            Future<Throwable> startStopWorker =
-                    service.submit( new StartStopLoad( keepGoing, onFailure, cluster, numberOfCores, numberOfEdges ) );
+            Future<Throwable> startStopWorker = service.submit(
+                    new StartStopLoad( fs, keepGoing, onFailure, cluster, numberOfCores, numberOfEdges ) );
             Future<Throwable> backupWorker = service.submit(
                     new BackupLoad( keepGoing, onFailure, cluster, numberOfCores, numberOfEdges, backupDirectory,
                             backupAddress ) );

diff --git a/...java/org/neo4j/causalclustering/stresstests/CatchupStoreCopyInteractionStressTesting.java b/...java/org/neo4j/causalclustering/stresstests/CatchupStoreCopyInteractionStressTesting.java
@@ -32,6 +32,8 @@
 
 import org.neo4j.causalclustering.discovery.Cluster;
 import org.neo4j.causalclustering.discovery.HazelcastDiscoveryServiceFactory;
+import org.neo4j.io.fs.DefaultFileSystemAbstraction;
+import org.neo4j.io.fs.FileSystemAbstraction;
 import org.neo4j.io.fs.FileUtils;
 import org.neo4j.kernel.impl.store.format.standard.StandardV3_0;
 
@@ -59,6 +61,8 @@ public class CatchupStoreCopyInteractionStressTesting
     private static final String DEFAULT_TX_PRUNE = "50 files";
     private static final String DEFAULT_WORKING_DIR = new File( getProperty( "java.io.tmpdir" ) ).getPath();
 
+    private final FileSystemAbstraction fs = new DefaultFileSystemAbstraction();
+
     @Test
     public void shouldBehaveCorrectlyUnderStress() throws Exception
     {
@@ -100,8 +104,8 @@ public void shouldBehaveCorrectlyUnderStress() throws Exception
             }
 
             Future<Throwable> workload = service.submit( new Workload( keepGoing, onFailure, cluster ) );
-            Future<Throwable> startStopWorker =
-                    service.submit( new StartStopLoad( keepGoing, onFailure, cluster, numberOfCores, numberOfEdges ) );
+            Future<Throwable> startStopWorker = service.submit(
+                    new StartStopLoad( fs, keepGoing, onFailure, cluster, numberOfCores, numberOfEdges ) );
             Future<Throwable> catchUpWorker = service.submit( new CatchUpLoad( keepGoing, onFailure, cluster ) );
 
             long timeout = durationInMinutes + 5;

diff --git a/stresstests/src/test/java/org/neo4j/causalclustering/stresstests/StartStopLoad.java b/stresstests/src/test/java/org/neo4j/causalclustering/stresstests/StartStopLoad.java
@@ -19,26 +19,66 @@
  */
 package org.neo4j.causalclustering.stresstests;
 
+import java.io.File;
 import java.util.concurrent.locks.LockSupport;
 import java.util.function.BooleanSupplier;
 
+import org.neo4j.causalclustering.catchup.storecopy.CopiedStoreRecovery;
+import org.neo4j.causalclustering.catchup.storecopy.TemporaryStoreDirectory;
 import org.neo4j.causalclustering.discovery.Cluster;
 import org.neo4j.causalclustering.discovery.ClusterMember;
+import org.neo4j.consistency.ConsistencyCheckService;
+import org.neo4j.io.fs.FileSystemAbstraction;
+import org.neo4j.io.pagecache.PageCache;
+import org.neo4j.kernel.configuration.Config;
+import org.neo4j.kernel.extension.KernelExtensions;
+import org.neo4j.kernel.impl.pagecache.StandalonePageCacheFactory;
+
+import static org.neo4j.consistency.ConsistencyCheckTool.runConsistencyCheckTool;
 
 class StartStopLoad extends RepeatUntilOnSelectedMemberCallable
 {
-    StartStopLoad( BooleanSupplier keepGoing, Runnable onFailure, Cluster cluster, int numberOfCores,
-            int numberOfEdges )
+    private final FileSystemAbstraction fs;
+
+    StartStopLoad( FileSystemAbstraction fs, BooleanSupplier keepGoing, Runnable onFailure, Cluster cluster,
+            int numberOfCores, int numberOfEdges )
     {
         super( keepGoing, onFailure, cluster, numberOfCores, numberOfEdges );
+        this.fs = fs;
     }
 
     @Override
     protected void doWorkOnMember( boolean isCore, int id )
     {
         ClusterMember member = isCore ? cluster.getCoreMemberById( id ) : cluster.getReadReplicaById( id );
+        String storeDir = member.database().getStoreDir();
+        KernelExtensions kernelExtensions =
+                member.database().getDependencyResolver().resolveDependency( KernelExtensions.class );
         member.shutdown();
+        assertStoreConsistent( storeDir, kernelExtensions );
         LockSupport.parkNanos( 5_000_000_000L );
         member.start();
     }
+
+    private void assertStoreConsistent( String storeDir, KernelExtensions kernelExtensions )
+    {
+        File fromDirectory = new File( storeDir );
+        File parent = fromDirectory.getParentFile();
+        try ( TemporaryStoreDirectory storeDirectory = new TemporaryStoreDirectory( fs, parent );
+                PageCache pageCache = StandalonePageCacheFactory.createPageCache( fs ) )
+        {
+            fs.copyRecursively( fromDirectory, storeDirectory.storeDir() );
+            new CopiedStoreRecovery( Config.defaults(), kernelExtensions.listFactories(),  pageCache )
+                    .recoverCopiedStore( storeDirectory.storeDir() );
+            ConsistencyCheckService.Result result = runConsistencyCheckTool( new String[]{storeDir} );
+            if ( !result.isSuccessful() )
+            {
+                throw new RuntimeException( "Not consistent database in " + storeDir );
+            }
+        }
+        catch ( Throwable e )
+        {
+            throw new RuntimeException( "Failed to run CC on " + storeDir, e );
+        }
+    }
 }