diff --git a/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/discovery/Cluster.java b/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/discovery/Cluster.java index e3f86be1f58b4..14b5dff2f9f4f 100644 --- a/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/discovery/Cluster.java +++ b/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/discovery/Cluster.java @@ -729,13 +729,29 @@ public Optional randomMember( boolean mustBeStarted ) } List eligible = members.collect( Collectors.toList() ); + return random( eligible ); + } + + public Optional randomCoreMember( boolean mustBeStarted ) + { + Stream members = coreMembers().stream(); - if ( eligible.size() == 0 ) + if ( mustBeStarted ) { - return Optional.empty(); + members = members.filter( m -> !m.isShutdown() ); } - int ordinal = ThreadLocalRandom.current().nextInt( eligible.size() ); - return Optional.of( eligible.get( ordinal ) ); + List eligible = members.collect( Collectors.toList() ); + return random( eligible ); + } + + private static Optional random( List list ) + { + if ( list.size() == 0 ) + { + return Optional.empty(); + } + int ordinal = ThreadLocalRandom.current().nextInt( list.size() ); + return Optional.of( list.get( ordinal ) ); } } diff --git a/stresstests/src/test/java/org/neo4j/causalclustering/stresstests/ClusterStressScenarioSmokeIT.java b/stresstests/src/test/java/org/neo4j/causalclustering/stresstests/ClusterStressScenarioSmokeIT.java index a5b7c747d0012..66e7af8ff0d0e 100644 --- a/stresstests/src/test/java/org/neo4j/causalclustering/stresstests/ClusterStressScenarioSmokeIT.java +++ b/stresstests/src/test/java/org/neo4j/causalclustering/stresstests/ClusterStressScenarioSmokeIT.java @@ -62,7 +62,7 @@ public void stressBackupRandomMemberAndStartStop() throws Exception @Test public void stressCatchupNewReadReplica() throws Exception { - config.workloads( Workloads.CreateNodesWithProperties, Workloads.CatchupNewReadReplica, Workloads.StartStopRandomMember ); + config.workloads( Workloads.CreateNodesWithProperties, Workloads.CatchupNewReadReplica, Workloads.StartStopRandomCore ); stressTest( config, fileSystem, pageCache ); } diff --git a/stresstests/src/test/java/org/neo4j/causalclustering/stresstests/RepeatOnRandomCore.java b/stresstests/src/test/java/org/neo4j/causalclustering/stresstests/RepeatOnRandomCore.java new file mode 100644 index 0000000000000..002b845b01ed8 --- /dev/null +++ b/stresstests/src/test/java/org/neo4j/causalclustering/stresstests/RepeatOnRandomCore.java @@ -0,0 +1,43 @@ +/* + * Copyright (c) 2002-2018 "Neo Technology," + * Network Engine for Objects in Lund AB [http://neotechnology.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +package org.neo4j.causalclustering.stresstests; + +import org.neo4j.causalclustering.discovery.Cluster; +import org.neo4j.causalclustering.discovery.ClusterMember; +import org.neo4j.helper.Workload; + +abstract class RepeatOnRandomCore extends Workload +{ + private final Cluster cluster; + + RepeatOnRandomCore( Control control, Resources resources ) + { + super( control ); + this.cluster = resources.cluster(); + } + + @Override + protected final void doWork() throws Exception + { + doWorkOnMember( cluster.randomCoreMember( true ).orElseThrow( IllegalStateException::new ) ); + } + + protected abstract void doWorkOnMember( ClusterMember member ) throws Exception; +} diff --git a/stresstests/src/test/java/org/neo4j/causalclustering/stresstests/StartStopRandomCore.java b/stresstests/src/test/java/org/neo4j/causalclustering/stresstests/StartStopRandomCore.java new file mode 100644 index 0000000000000..c0099b9883ee1 --- /dev/null +++ b/stresstests/src/test/java/org/neo4j/causalclustering/stresstests/StartStopRandomCore.java @@ -0,0 +1,87 @@ +/* + * Copyright (c) 2002-2018 "Neo Technology," + * Network Engine for Objects in Lund AB [http://neotechnology.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +package org.neo4j.causalclustering.stresstests; + +import java.io.File; +import java.io.PrintStream; + +import org.neo4j.causalclustering.catchup.storecopy.CopiedStoreRecovery; +import org.neo4j.causalclustering.catchup.storecopy.TemporaryStoreDirectory; +import org.neo4j.causalclustering.discovery.ClusterMember; +import org.neo4j.consistency.ConsistencyCheckService; +import org.neo4j.io.fs.FileSystemAbstraction; +import org.neo4j.io.pagecache.PageCache; +import org.neo4j.io.pagecache.impl.muninn.StandalonePageCacheFactory; +import org.neo4j.kernel.configuration.Config; +import org.neo4j.kernel.extension.KernelExtensions; +import org.neo4j.logging.Log; + +import static org.neo4j.consistency.ConsistencyCheckTool.runConsistencyCheckTool; +import static org.neo4j.io.NullOutputStream.NULL_OUTPUT_STREAM; + +class StartStopRandomCore extends RepeatOnRandomCore +{ + private final FileSystemAbstraction fs; + private final PageCache pageCache; + private final Log log; + + StartStopRandomCore( Control control, Resources resources ) + { + super( control, resources ); + this.fs = resources.fileSystem(); + this.pageCache = resources.pageCache(); + this.log = resources.logProvider().getLog( getClass() ); + } + + @Override + protected void doWorkOnMember( ClusterMember member ) throws InterruptedException + { + File storeDir = member.database().getStoreDir(); + KernelExtensions kernelExtensions = member.database().getDependencyResolver().resolveDependency( KernelExtensions.class ); + log.info( "Stopping: " + member ); + member.shutdown(); + assertStoreConsistent( storeDir, kernelExtensions ); + Thread.sleep( 5000 ); + log.info( "Starting: " + member ); + member.start(); + } + + private void assertStoreConsistent( File storeDir, KernelExtensions kernelExtensions ) + { + File parent = storeDir.getParentFile(); + try ( TemporaryStoreDirectory storeDirectory = new TemporaryStoreDirectory( fs, pageCache, parent ); + PageCache pageCache = StandalonePageCacheFactory.createPageCache( fs ) ) + { + fs.copyRecursively( storeDir, storeDirectory.storeDir() ); + new CopiedStoreRecovery( Config.defaults(), kernelExtensions.listFactories(), pageCache ) + .recoverCopiedStore( storeDirectory.storeDir() ); + ConsistencyCheckService.Result result = runConsistencyCheckTool( new String[]{storeDir.getAbsolutePath()}, + new PrintStream( NULL_OUTPUT_STREAM ), new PrintStream( NULL_OUTPUT_STREAM ) ); + if ( !result.isSuccessful() ) + { + throw new RuntimeException( "Not consistent database in " + storeDir ); + } + } + catch ( Throwable e ) + { + throw new RuntimeException( "Failed to run CC on " + storeDir, e ); + } + } +} diff --git a/stresstests/src/test/java/org/neo4j/causalclustering/stresstests/Workloads.java b/stresstests/src/test/java/org/neo4j/causalclustering/stresstests/Workloads.java index b5581f1390a5f..fe1d2441496f1 100644 --- a/stresstests/src/test/java/org/neo4j/causalclustering/stresstests/Workloads.java +++ b/stresstests/src/test/java/org/neo4j/causalclustering/stresstests/Workloads.java @@ -39,6 +39,14 @@ Workload create( Control control, Resources resources, Config config ) return new StartStopRandomMember( control, resources ); } }, + StartStopRandomCore + { + @Override + Workload create( Control control, Resources resources, Config config ) + { + return new StartStopRandomCore( control, resources ); + } + }, BackupRandomMember { @Override