From 71a3fd2ebd2652dce2fd2b9678bffaed613eec27 Mon Sep 17 00:00:00 2001 From: Alex Heneveld Date: Wed, 13 Sep 2017 17:52:08 +0100 Subject: [PATCH 1/2] add load test without persistence so we can see what else is chewing up cpu --- .../SimulatedVanillaSoftwareProcessImpl.java | 9 +++ .../brooklyn/qa/load/AbstractLoadTest.java | 8 ++- .../qa/load/LoadWithoutPersistenceTest.java | 65 +++++++++++++++++++ 3 files changed, 80 insertions(+), 2 deletions(-) create mode 100644 qa/src/test/java/org/apache/brooklyn/qa/load/LoadWithoutPersistenceTest.java diff --git a/qa/src/main/java/org/apache/brooklyn/qa/load/SimulatedVanillaSoftwareProcessImpl.java b/qa/src/main/java/org/apache/brooklyn/qa/load/SimulatedVanillaSoftwareProcessImpl.java index e5aea80f6..6b1d09646 100644 --- a/qa/src/main/java/org/apache/brooklyn/qa/load/SimulatedVanillaSoftwareProcessImpl.java +++ b/qa/src/main/java/org/apache/brooklyn/qa/load/SimulatedVanillaSoftwareProcessImpl.java @@ -216,5 +216,14 @@ public void launch() { // no-op } } + + @Override + public boolean isRunning() { + if (Boolean.TRUE.equals(entity.getConfig(USE_SSH_MONITORING))) { + return super.isRunning(); + } else { + return true; + } + } } } diff --git a/qa/src/test/java/org/apache/brooklyn/qa/load/AbstractLoadTest.java b/qa/src/test/java/org/apache/brooklyn/qa/load/AbstractLoadTest.java index c07f32924..56cd756a2 100644 --- a/qa/src/test/java/org/apache/brooklyn/qa/load/AbstractLoadTest.java +++ b/qa/src/test/java/org/apache/brooklyn/qa/load/AbstractLoadTest.java @@ -161,8 +161,8 @@ protected ManagementContext setUpPlatform() { // Create management node persistenceDir = Files.createTempDir(); launcher = BrooklynLauncher.newInstance() - .persistMode(PersistMode.CLEAN) - .highAvailabilityMode(HighAvailabilityMode.MASTER) + .persistMode(doPersistence() ? PersistMode.CLEAN : PersistMode.DISABLED) + .highAvailabilityMode(doPersistence() ? HighAvailabilityMode.MASTER : HighAvailabilityMode.DISABLED) .persistenceDir(persistenceDir) .start(); @@ -171,6 +171,10 @@ protected ManagementContext setUpPlatform() { return launcher.getServerDetails().getManagementContext(); } + + protected boolean doPersistence() { + return true; + } @Override protected void tearDownPlatform() { diff --git a/qa/src/test/java/org/apache/brooklyn/qa/load/LoadWithoutPersistenceTest.java b/qa/src/test/java/org/apache/brooklyn/qa/load/LoadWithoutPersistenceTest.java new file mode 100644 index 000000000..c69fc4fa2 --- /dev/null +++ b/qa/src/test/java/org/apache/brooklyn/qa/load/LoadWithoutPersistenceTest.java @@ -0,0 +1,65 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.brooklyn.qa.load; + +import org.apache.brooklyn.entity.software.base.VanillaSoftwareProcess; +import org.apache.brooklyn.util.time.Duration; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.testng.annotations.Test; + +import com.google.common.base.Predicates; +import com.google.common.base.Stopwatch; + +public class LoadWithoutPersistenceTest extends AbstractLoadTest { + + private static final Logger log = LoggerFactory.getLogger(LoadWithoutPersistenceTest.class); + + @Override + protected boolean doPersistence() { + return false; + } + + /** + * Creates many SSH simulated external monitor apps, to ensure resource usage not extreme. + * + * Long-term target is 2500 VMs under management. + * Until we reach that point, we can partition the load across multiple (separate) brooklyn management nodes. + */ + @Test(groups="Acceptance") + public void testManyAppsExternallyMonitored() throws Exception { + // TODO Getting ssh error ("Server closed connection during identification exchange") + // with only two cycles (i.e. 20 entities). + // + // The ssh activity is from `SoftwareProcessImpl.waitForEntityStart`, which calls + // `VanillaSoftwareProcessSshDriver.isRunning`. + final int TOTAL_APPS = 500; // target is 2500 VMs; each blueprint has 2 VanillaSoftwareProcess + final int NUM_APPS_PER_BATCH = 10; + Stopwatch startTime = Stopwatch.createStarted(); + super.runLocalhostManyApps(new TestConfig(this) + .execSshOnStart(false) // getting ssh errors otherwise! + .useSshMonitoring(false) // getting ssh errors otherwise! + .simulateExternalMonitor(Predicates.instanceOf(VanillaSoftwareProcess.class), 5, Duration.ONE_SECOND) + .clusterSize(5) + .totalApps(TOTAL_APPS, NUM_APPS_PER_BATCH) + .sleepBetweenBatch(Duration.seconds(0))); + log.info("Created "+TOTAL_APPS+" apps in "+Duration.of(startTime)); + } + +} From 4babb503460bbd938352b603c397214e53a1ef08 Mon Sep 17 00:00:00 2001 From: Alex Heneveld Date: Wed, 13 Sep 2017 17:52:43 +0100 Subject: [PATCH 2/2] set highlights on software-specific enrichers and policies --- .../entity/network/bind/PrefixAndIdEnricher.java | 2 ++ .../nosql/mongodb/sharding/MongoDBRouterClusterImpl.java | 9 +++++++++ .../brooklyn/entity/dns/AbstractGeoDnsServiceImpl.java | 1 + .../brooklyn/entity/proxy/AbstractControllerImpl.java | 1 + .../proxy/AbstractNonProvisionedControllerImpl.java | 3 +++ .../brooklyn/entity/proxy/nginx/NginxControllerImpl.java | 1 + 6 files changed, 17 insertions(+) diff --git a/software/network/src/test/java/org/apache/brooklyn/entity/network/bind/PrefixAndIdEnricher.java b/software/network/src/test/java/org/apache/brooklyn/entity/network/bind/PrefixAndIdEnricher.java index cf035308d..87a72e887 100644 --- a/software/network/src/test/java/org/apache/brooklyn/entity/network/bind/PrefixAndIdEnricher.java +++ b/software/network/src/test/java/org/apache/brooklyn/entity/network/bind/PrefixAndIdEnricher.java @@ -37,6 +37,7 @@ public class PrefixAndIdEnricher extends AbstractEnricher { public static final ConfigKey PREFIX = ConfigKeys.newStringConfigKey( "prefixandid.prefix", "Sets SENSOR to prefix+entity id"); + @SuppressWarnings("serial") public static final ConfigKey> MONITOR = ConfigKeys.newConfigKey(new TypeToken>() {}, "prefixandid.attributetomonitor", "Changes on this sensor are monitored and the prefix/id republished"); @@ -52,6 +53,7 @@ public void onEvent(SensorEvent event) { entity.sensors().set(SENSOR, getConfig(PREFIX) + entity.getId()); } }); + highlightTriggers(getConfig(MONITOR), null); } } diff --git a/software/nosql/src/main/java/org/apache/brooklyn/entity/nosql/mongodb/sharding/MongoDBRouterClusterImpl.java b/software/nosql/src/main/java/org/apache/brooklyn/entity/nosql/mongodb/sharding/MongoDBRouterClusterImpl.java index 48b501c19..ccff4e23e 100644 --- a/software/nosql/src/main/java/org/apache/brooklyn/entity/nosql/mongodb/sharding/MongoDBRouterClusterImpl.java +++ b/software/nosql/src/main/java/org/apache/brooklyn/entity/nosql/mongodb/sharding/MongoDBRouterClusterImpl.java @@ -30,12 +30,16 @@ import org.apache.brooklyn.core.entity.trait.Startable; import org.apache.brooklyn.entity.group.AbstractMembershipTrackingPolicy; import org.apache.brooklyn.entity.group.DynamicClusterImpl; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import com.google.common.collect.ImmutableList; import com.google.common.collect.Iterables; public class MongoDBRouterClusterImpl extends DynamicClusterImpl implements MongoDBRouterCluster { + private static final Logger log = LoggerFactory.getLogger(MongoDBRouterClusterImpl.class); + @Override public void init() { super.init(); @@ -56,12 +60,17 @@ public void start(Collection locations) { public static class MemberTrackingPolicy extends AbstractMembershipTrackingPolicy { @Override protected void onEntityEvent(EventType type, Entity member) { + defaultHighlightAction(type, entity); ((MongoDBRouterClusterImpl)super.entity).setAnyRouter(); } @Override protected void onEntityRemoved(Entity member) { + // TODO shouldn't be invoked - remove + log.warn("Removal handler should be hidden by event handler", new Throwable("Trace for unexpected mongo node handler")); ((MongoDBRouterClusterImpl)super.entity).setAnyRouter(); } @Override protected void onEntityChange(Entity member) { + // TODO shouldn't be invoked - remove + log.warn("Change handler should be hidden by event handler", new Throwable("Trace for unexpected mongo node handler")); ((MongoDBRouterClusterImpl)super.entity).setAnyRouter(); } } diff --git a/software/webapp/src/main/java/org/apache/brooklyn/entity/dns/AbstractGeoDnsServiceImpl.java b/software/webapp/src/main/java/org/apache/brooklyn/entity/dns/AbstractGeoDnsServiceImpl.java index 0f96c8714..d94d5510f 100644 --- a/software/webapp/src/main/java/org/apache/brooklyn/entity/dns/AbstractGeoDnsServiceImpl.java +++ b/software/webapp/src/main/java/org/apache/brooklyn/entity/dns/AbstractGeoDnsServiceImpl.java @@ -180,6 +180,7 @@ protected void endTracker() { public static class MemberTrackingPolicy extends AbstractMembershipTrackingPolicy { @Override protected void onEntityEvent(EventType type, Entity entity) { + defaultHighlightAction(type, entity); ((AbstractGeoDnsServiceImpl)super.entity).refreshGroupMembership(); } } diff --git a/software/webapp/src/main/java/org/apache/brooklyn/entity/proxy/AbstractControllerImpl.java b/software/webapp/src/main/java/org/apache/brooklyn/entity/proxy/AbstractControllerImpl.java index c24b2bc96..34c324e16 100644 --- a/software/webapp/src/main/java/org/apache/brooklyn/entity/proxy/AbstractControllerImpl.java +++ b/software/webapp/src/main/java/org/apache/brooklyn/entity/proxy/AbstractControllerImpl.java @@ -154,6 +154,7 @@ protected void removeServerPoolMemberTrackingPolicy() { public static class ServerPoolMemberTrackerPolicy extends AbstractMembershipTrackingPolicy { @Override protected void onEntityEvent(EventType type, Entity entity) { + defaultHighlightAction(type, entity); // relies on policy-rebind injecting the implementation rather than the dynamic-proxy ((AbstractControllerImpl)super.entity).onServerPoolMemberChanged(entity); } diff --git a/software/webapp/src/main/java/org/apache/brooklyn/entity/proxy/AbstractNonProvisionedControllerImpl.java b/software/webapp/src/main/java/org/apache/brooklyn/entity/proxy/AbstractNonProvisionedControllerImpl.java index c26235a6f..7ad722462 100644 --- a/software/webapp/src/main/java/org/apache/brooklyn/entity/proxy/AbstractNonProvisionedControllerImpl.java +++ b/software/webapp/src/main/java/org/apache/brooklyn/entity/proxy/AbstractNonProvisionedControllerImpl.java @@ -51,6 +51,7 @@ import com.google.common.collect.Iterables; import com.google.common.collect.Maps; +/** For use by downstream load-balancers. */ public abstract class AbstractNonProvisionedControllerImpl extends AbstractEntity implements AbstractNonProvisionedController { private static final Logger LOG = LoggerFactory.getLogger(AbstractNonProvisionedControllerImpl.class); @@ -66,6 +67,7 @@ public AbstractNonProvisionedControllerImpl() { public static class MemberTrackingPolicy extends AbstractMembershipTrackingPolicy { @Override protected void onEntityEvent(EventType type, Entity member) { + defaultHighlightAction(type, entity); ((AbstractNonProvisionedControllerImpl)super.entity).onServerPoolMemberChanged(member); } } @@ -216,6 +218,7 @@ protected void removeServerPoolMemberTrackingPolicy() { public static class ServerPoolMemberTrackerPolicy extends AbstractMembershipTrackingPolicy { @Override protected void onEntityEvent(EventType type, Entity entity) { + defaultHighlightAction(type, entity); // relies on policy-rebind injecting the implementation rather than the dynamic-proxy ((AbstractNonProvisionedControllerImpl)super.entity).onServerPoolMemberChanged(entity); } diff --git a/software/webapp/src/main/java/org/apache/brooklyn/entity/proxy/nginx/NginxControllerImpl.java b/software/webapp/src/main/java/org/apache/brooklyn/entity/proxy/nginx/NginxControllerImpl.java index 775690955..979f915e0 100644 --- a/software/webapp/src/main/java/org/apache/brooklyn/entity/proxy/nginx/NginxControllerImpl.java +++ b/software/webapp/src/main/java/org/apache/brooklyn/entity/proxy/nginx/NginxControllerImpl.java @@ -178,6 +178,7 @@ protected void removeUrlMappingsMemberTrackerPolicy() { public static class UrlMappingsMemberTrackerPolicy extends AbstractMembershipTrackingPolicy { @Override protected void onEntityEvent(EventType type, Entity entity) { + defaultHighlightAction(type, entity); // relies on policy-rebind injecting the implementation rather than the dynamic-proxy ((NginxControllerImpl)super.entity).updateNeeded(); }