From ec1c53d0930c91cb58a65fc5486cce90fab976dd Mon Sep 17 00:00:00 2001 From: Davide Grohmann Date: Fri, 16 Sep 2016 11:41:59 +0200 Subject: [PATCH] Tweak configurations in stress tests Small cleanups and sharing of code --- ...ckupStoreCopyInteractionStressTesting.java | 30 ++++------ ...chupStoreCopyInteractionStressTesting.java | 15 ++--- .../stresstests/ClusterConfiguration.java | 57 +++++++++++++++++++ .../coreedge/stresstests/StartStopLoad.java | 2 +- .../neo4j/coreedge/stresstests/Workload.java | 11 +++- 5 files changed, 85 insertions(+), 30 deletions(-) create mode 100644 stresstests/src/test/java/org/neo4j/coreedge/stresstests/ClusterConfiguration.java diff --git a/stresstests/src/test/java/org/neo4j/coreedge/stresstests/BackupStoreCopyInteractionStressTesting.java b/stresstests/src/test/java/org/neo4j/coreedge/stresstests/BackupStoreCopyInteractionStressTesting.java index e69f02b51c38f..3e2ae9ada7726 100644 --- a/stresstests/src/test/java/org/neo4j/coreedge/stresstests/BackupStoreCopyInteractionStressTesting.java +++ b/stresstests/src/test/java/org/neo4j/coreedge/stresstests/BackupStoreCopyInteractionStressTesting.java @@ -33,8 +33,6 @@ import java.util.function.BooleanSupplier; import java.util.function.IntFunction; -import org.neo4j.backup.OnlineBackupSettings; -import org.neo4j.coreedge.core.CoreEdgeClusterSettings; import org.neo4j.coreedge.discovery.Cluster; import org.neo4j.coreedge.discovery.HazelcastDiscoveryServiceFactory; import org.neo4j.helpers.AdvertisedSocketAddress; @@ -45,12 +43,13 @@ import static java.lang.Integer.parseInt; import static java.lang.Long.parseLong; import static java.lang.System.getProperty; -import static java.util.Collections.emptyMap; import static org.junit.Assert.assertTrue; import static org.neo4j.StressTestingHelper.ensureExistsAndEmpty; import static org.neo4j.StressTestingHelper.fromEnv; +import static org.neo4j.coreedge.stresstests.ClusterConfiguration.configureBackup; +import static org.neo4j.coreedge.stresstests.ClusterConfiguration.configureRaftLogRotationAndPruning; +import static org.neo4j.coreedge.stresstests.ClusterConfiguration.configureTxLogRotationAndPruning; import static org.neo4j.function.Suppliers.untilTimeExpired; -import static org.neo4j.kernel.configuration.Settings.TRUE; public class BackupStoreCopyInteractionStressTesting { @@ -83,18 +82,19 @@ public void shouldBehaveCorrectlyUnderStress() throws Exception BiFunction backupAddress = ( isCore, id ) -> new AdvertisedSocketAddress( "localhost", (isCore ? baseCoreBackupPort : baseEdgeBackupPort) + id ); - Map coreParams = new HashMap<>(); - coreParams.put( CoreEdgeClusterSettings.raft_log_rotation_size.name(), "1K" ); - coreParams.put( CoreEdgeClusterSettings.raft_log_pruning_frequency.name(), "1s" ); - coreParams.put( CoreEdgeClusterSettings.raft_log_pruning_strategy.name(), "keep_none" ); + Map coreParams = + configureRaftLogRotationAndPruning( configureTxLogRotationAndPruning( new HashMap<>() ) ); + Map edgeParams = configureTxLogRotationAndPruning( new HashMap<>() ); - Map> paramsPerCoreInstance = configureBackup( (id) -> backupAddress.apply( true, id ) ); - Map> paramsPerEdgeInstance = configureBackup( (id) -> backupAddress.apply( false, id ) ); + Map> instanceCoreParams = + configureBackup( new HashMap<>(), id -> backupAddress.apply( true, id ) ); + Map> instanceEdgeParams = + configureBackup( new HashMap<>(), id -> backupAddress.apply( false, id ) ); HazelcastDiscoveryServiceFactory discoveryServiceFactory = new HazelcastDiscoveryServiceFactory(); Cluster cluster = new Cluster( clusterDirectory, numberOfCores, numberOfEdges, discoveryServiceFactory, coreParams, - paramsPerCoreInstance, emptyMap(), paramsPerEdgeInstance, StandardV3_0.NAME ); + instanceCoreParams, edgeParams, instanceEdgeParams, StandardV3_0.NAME ); AtomicBoolean stopTheWorld = new AtomicBoolean(); BooleanSupplier keepGoing = @@ -123,12 +123,4 @@ public void shouldBehaveCorrectlyUnderStress() throws Exception FileUtils.deleteRecursively( clusterDirectory ); FileUtils.deleteRecursively( backupDirectory ); } - - private static Map> configureBackup( IntFunction address ) - { - Map> settings = new HashMap<>(); - settings.put( OnlineBackupSettings.online_backup_enabled.name(), id -> TRUE ); - settings.put( OnlineBackupSettings.online_backup_server.name(), id -> address.apply( id ).toString() ); - return settings; - } } diff --git a/stresstests/src/test/java/org/neo4j/coreedge/stresstests/CatchupStoreCopyInteractionStressTesting.java b/stresstests/src/test/java/org/neo4j/coreedge/stresstests/CatchupStoreCopyInteractionStressTesting.java index 79d179e52b770..ed32b230204c0 100644 --- a/stresstests/src/test/java/org/neo4j/coreedge/stresstests/CatchupStoreCopyInteractionStressTesting.java +++ b/stresstests/src/test/java/org/neo4j/coreedge/stresstests/CatchupStoreCopyInteractionStressTesting.java @@ -22,7 +22,6 @@ import org.junit.Test; import java.io.File; -import java.util.Collections; import java.util.HashMap; import java.util.Map; import java.util.concurrent.ExecutorService; @@ -31,10 +30,7 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import java.util.function.BooleanSupplier; -import java.util.function.Consumer; -import java.util.function.IntFunction; -import org.neo4j.coreedge.core.CoreEdgeClusterSettings; import org.neo4j.coreedge.discovery.Cluster; import org.neo4j.coreedge.discovery.HazelcastDiscoveryServiceFactory; import org.neo4j.io.fs.FileUtils; @@ -47,6 +43,8 @@ import static org.junit.Assert.assertTrue; import static org.neo4j.StressTestingHelper.ensureExistsAndEmpty; import static org.neo4j.StressTestingHelper.fromEnv; +import static org.neo4j.coreedge.stresstests.ClusterConfiguration.configureRaftLogRotationAndPruning; +import static org.neo4j.coreedge.stresstests.ClusterConfiguration.configureTxLogRotationAndPruning; import static org.neo4j.function.Suppliers.untilTimeExpired; public class CatchupStoreCopyInteractionStressTesting @@ -70,15 +68,14 @@ public void shouldBehaveCorrectlyUnderStress() throws Exception File clusterDirectory = ensureExistsAndEmpty( new File( workingDirectory, "cluster" ) ); - Map coreParams = new HashMap<>(); - coreParams.put( CoreEdgeClusterSettings.raft_log_rotation_size.name(), "1K" ); - coreParams.put( CoreEdgeClusterSettings.raft_log_pruning_frequency.name(), "1s" ); - coreParams.put( CoreEdgeClusterSettings.raft_log_pruning_strategy.name(), "keep_none" ); + Map coreParams = + configureRaftLogRotationAndPruning( configureTxLogRotationAndPruning( new HashMap<>() ) ); + Map edgeParams = configureTxLogRotationAndPruning( new HashMap<>() ); HazelcastDiscoveryServiceFactory discoveryServiceFactory = new HazelcastDiscoveryServiceFactory(); Cluster cluster = new Cluster( clusterDirectory, numberOfCores, numberOfEdges, discoveryServiceFactory, coreParams, - emptyMap(), emptyMap(), emptyMap(), StandardV3_0.NAME ); + emptyMap(), edgeParams, emptyMap(), StandardV3_0.NAME ); AtomicBoolean stopTheWorld = new AtomicBoolean(); BooleanSupplier keepGoing = diff --git a/stresstests/src/test/java/org/neo4j/coreedge/stresstests/ClusterConfiguration.java b/stresstests/src/test/java/org/neo4j/coreedge/stresstests/ClusterConfiguration.java new file mode 100644 index 0000000000000..536fa85d5dda2 --- /dev/null +++ b/stresstests/src/test/java/org/neo4j/coreedge/stresstests/ClusterConfiguration.java @@ -0,0 +1,57 @@ +/* + * Copyright (c) 2002-2016 "Neo Technology," + * Network Engine for Objects in Lund AB [http://neotechnology.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +package org.neo4j.coreedge.stresstests; + +import java.util.Map; +import java.util.function.IntFunction; + +import org.neo4j.backup.OnlineBackupSettings; +import org.neo4j.coreedge.core.CoreEdgeClusterSettings; +import org.neo4j.graphdb.factory.GraphDatabaseSettings; +import org.neo4j.helpers.SocketAddress; +import org.neo4j.kernel.configuration.Settings; + +import static org.neo4j.kernel.configuration.Settings.TRUE; + +class ClusterConfiguration +{ + static Map configureRaftLogRotationAndPruning( Map settings ) + { + settings.put( CoreEdgeClusterSettings.raft_log_rotation_size.name(), "1K" ); + settings.put( CoreEdgeClusterSettings.raft_log_pruning_frequency.name(), "250ms" ); + settings.put( CoreEdgeClusterSettings.raft_log_pruning_strategy.name(), "keep_none" ); + return settings; + } + + static Map configureTxLogRotationAndPruning( Map settings ) + { + settings.put( GraphDatabaseSettings.keep_logical_logs.name(), Settings.FALSE ); + settings.put( GraphDatabaseSettings.logical_log_rotation_threshold.name(), "1M" ); + return settings; + } + + static Map> configureBackup( Map> settings, + IntFunction address ) + { + settings.put( OnlineBackupSettings.online_backup_enabled.name(), id -> TRUE ); + settings.put( OnlineBackupSettings.online_backup_server.name(), id -> address.apply( id ).toString() ); + return settings; + } +} diff --git a/stresstests/src/test/java/org/neo4j/coreedge/stresstests/StartStopLoad.java b/stresstests/src/test/java/org/neo4j/coreedge/stresstests/StartStopLoad.java index c89b8b1c86e84..d4b60ecb529cd 100644 --- a/stresstests/src/test/java/org/neo4j/coreedge/stresstests/StartStopLoad.java +++ b/stresstests/src/test/java/org/neo4j/coreedge/stresstests/StartStopLoad.java @@ -37,7 +37,7 @@ protected boolean doWorkOnMember( boolean isCore, int id ) { ClusterMember member = isCore ? cluster.getCoreMemberById( id ) : cluster.getEdgeMemberById( id ); member.shutdown(); - LockSupport.parkNanos( 2_000_000_000 ); + LockSupport.parkNanos( 5_000_000_000L ); member.start(); return true; } diff --git a/stresstests/src/test/java/org/neo4j/coreedge/stresstests/Workload.java b/stresstests/src/test/java/org/neo4j/coreedge/stresstests/Workload.java index 26cb89cd4cb73..7df98964c69e3 100644 --- a/stresstests/src/test/java/org/neo4j/coreedge/stresstests/Workload.java +++ b/stresstests/src/test/java/org/neo4j/coreedge/stresstests/Workload.java @@ -24,6 +24,7 @@ import org.neo4j.coreedge.discovery.Cluster; import org.neo4j.graphdb.DatabaseShutdownException; +import org.neo4j.graphdb.Node; class Workload extends RepeatUntilCallable { @@ -42,7 +43,15 @@ protected boolean doWork() { cluster.coreTx( ( db, tx ) -> { - db.createNode(); + Node node = db.createNode(); + node.setProperty( "prop1", "let's add some data here so the transaction logs rotate more often..." ); + node.setProperty( "prop2", "let's add some data here so the transaction logs rotate more often..." ); + node.setProperty( "prop3", "let's add some data here so the transaction logs rotate more often..." ); + node.setProperty( "prop4", "let's add some data here so the transaction logs rotate more often..." ); + node.setProperty( "prop5", "let's add some data here so the transaction logs rotate more often..." ); + node.setProperty( "prop6", "let's add some data here so the transaction logs rotate more often..." ); + node.setProperty( "prop7", "let's add some data here so the transaction logs rotate more often..." ); + node.setProperty( "prop8", "let's add some data here so the transaction logs rotate more often..." ); tx.success(); } ); }