Skip to content
Permalink
Browse files
BROOKLYN-212: AutoScaling doesn’t retry if InsufficientCapacity
- Adds Resizable.InsufficientCapacityException, thrown by
  Resizable.resize() if could not grow at all.
- DynamicCluster catches NoMachinesAvailableException, and rethrows
  as InsufficientCapacityException in resize().
- AutoScalerPolicy catches InsufficientCapacityException, and sets
  insufficientCapacityHighWaterMark to record the max size it can get
  to. Does not try again to resize above that, unless the highWaterMark
  is explicitly cleared by reconfiguring that config value.
- Tests:
  - Changes TestCluster to include history of sizes and desiredSizes
  - Changes TestCluster, so can throw InsufficientCapacityException
    when gets to a particular size.
  - Test for DynamicCluster throwing InsufficientCapacityException
  - Test for AutoScalerPolicyMetricTest, to not resize above the failure
    level again.
  - Test for AutoScalerPolicyNoMoreMachinesTest, for when BYON location
    has run out of machines in a DynamicCluster.
  • Loading branch information
aledsage committed Jan 14, 2016
1 parent dd3b8e8 commit a91889d467848ceefa88855fd615afe9bca94931
Showing 10 changed files with 421 additions and 14 deletions.
@@ -31,13 +31,29 @@
*/
public interface Resizable {

/**
* Indicates that resizing up (at all) is not possible, because there is insufficient capacity.
*/
public static class InsufficientCapacityException extends RuntimeException {
private static final long serialVersionUID = 953230498564942446L;

public InsufficientCapacityException(String msg) {
super(msg);
}
public InsufficientCapacityException(String msg, Throwable cause) {
super(msg, cause);
}
}

MethodEffector<Integer> RESIZE = new MethodEffector<Integer>(Resizable.class, "resize");

/**
* Grow or shrink this entity to the desired size.
*
* @param desiredSize the new size of the entity group.
* @return the new size of the group.
*
* @throws InsufficientCapacityException If the request was to grow, but there is no capacity to grow at all
*/
@Effector(description="Changes the size of the entity (e.g. the number of nodes in a cluster)")
Integer resize(@EffectorParam(name="desiredSize", description="The new size of the cluster") Integer desiredSize);
@@ -38,6 +38,7 @@
import org.apache.brooklyn.api.entity.Group;
import org.apache.brooklyn.api.location.Location;
import org.apache.brooklyn.api.location.MachineProvisioningLocation;
import org.apache.brooklyn.api.location.NoMachinesAvailableException;
import org.apache.brooklyn.api.mgmt.Task;
import org.apache.brooklyn.api.policy.Policy;
import org.apache.brooklyn.api.sensor.AttributeSensor;
@@ -50,6 +51,7 @@
import org.apache.brooklyn.core.entity.lifecycle.Lifecycle;
import org.apache.brooklyn.core.entity.lifecycle.ServiceStateLogic;
import org.apache.brooklyn.core.entity.lifecycle.ServiceStateLogic.ServiceProblemsLogic;
import org.apache.brooklyn.core.entity.trait.Resizable;
import org.apache.brooklyn.core.entity.trait.Startable;
import org.apache.brooklyn.core.entity.trait.StartableMethods;
import org.apache.brooklyn.core.location.Locations;
@@ -518,7 +520,20 @@ public Integer resize(Integer desiredSize) {
} else {
if (LOG.isDebugEnabled()) LOG.debug("Resize no-op {} from {} to {}", new Object[] {this, originalSize, desiredSize});
}
resizeByDelta(delta);
// If we managed to grow at all, then expect no exception.
// Otherwise, if failed because NoMachinesAvailable, then propagate as InsufficientCapacityException.
// This tells things like the AutoScalerPolicy to not keep retrying.
try {
resizeByDelta(delta);
} catch (Exception e) {
Exceptions.propagateIfFatal(e);
NoMachinesAvailableException nmae = Exceptions.getFirstThrowableOfType(e, NoMachinesAvailableException.class);
if (nmae != null) {
throw new Resizable.InsufficientCapacityException("Failed to resize", e);
} else {
throw Exceptions.propagate(e);
}
}
}
return getCurrentSize();
}
@@ -669,7 +684,7 @@ public Collection<Entity> resizeByDelta(int delta) {
}
}

/** <strong>Note</strong> for sub-clases; this method can be called while synchronized on {@link #mutex}. */
/** <strong>Note</strong> for sub-classes; this method can be called while synchronized on {@link #mutex}. */
protected Collection<Entity> grow(int delta) {
Preconditions.checkArgument(delta > 0, "Must call grow with positive delta.");

@@ -697,7 +712,15 @@ protected Collection<Entity> grow(int delta) {
}

// create and start the entities
return addInEachLocation(chosenLocations, ImmutableMap.of()).getWithError();
ReferenceWithError<Collection<Entity>> result = addInEachLocation(chosenLocations, ImmutableMap.of());

// If any entities were created, return them (even if we didn't manage to create them all).
// Otherwise, propagate any error that happened.
if (result.get().size() > 0) {
return result.get();
} else {
return result.getWithError();
}
}

/** <strong>Note</strong> for sub-clases; this method can be called while synchronized on {@link #mutex}. */
@@ -60,7 +60,7 @@ public interface FailingEntity extends TestEntity {
ConfigKey<Predicate<? super FailingEntity>> FAIL_ON_RESTART_CONDITION = (ConfigKey) ConfigKeys.newConfigKey(Predicate.class, "failOnRestartCondition", "Whether to throw exception on call to restart", null);

@SetFromFlag("exceptionClazz")
ConfigKey<Class<? extends RuntimeException>> EXCEPTION_CLAZZ = (ConfigKey) ConfigKeys.newConfigKey(Class.class, "exceptionClazz", "Type of exception to throw", IllegalStateException.class);
ConfigKey<Class<? extends Exception>> EXCEPTION_CLAZZ = (ConfigKey) ConfigKeys.newConfigKey(Class.class, "exceptionClazz", "Type of exception to throw", IllegalStateException.class);

@SetFromFlag("execOnFailure")
ConfigKey<Function<? super FailingEntity,?>> EXEC_ON_FAILURE = (ConfigKey) ConfigKeys.newConfigKey(Function.class, "execOnFailure", "Callback to execute before throwing an exception, on any failure", Functions.identity());
@@ -79,7 +79,12 @@ private RuntimeException fail(final String msg) {

private RuntimeException newException(String msg) {
try {
return getConfig(EXCEPTION_CLAZZ).getConstructor(String.class).newInstance("Simulating entity stop failure for test");
Exception result = getConfig(EXCEPTION_CLAZZ).getConstructor(String.class).newInstance("Simulating entity stop failure for test");
if (!(result instanceof RuntimeException)) {
return new RuntimeException("wrapping", result);
} else {
return (RuntimeException)result;
}
} catch (Exception e) {
throw Exceptions.propagate(e);
}
@@ -18,13 +18,23 @@
*/
package org.apache.brooklyn.core.test.entity;

import java.util.List;

import org.apache.brooklyn.api.entity.EntityLocal;
import org.apache.brooklyn.api.entity.ImplementedBy;
import org.apache.brooklyn.config.ConfigKey;
import org.apache.brooklyn.core.config.ConfigKeys;
import org.apache.brooklyn.entity.group.DynamicCluster;

/**
* Mock cluster entity for testing.
*/
@ImplementedBy(TestClusterImpl.class)
public interface TestCluster extends DynamicCluster, EntityLocal {

ConfigKey<Integer> MAX_SIZE = ConfigKeys.newIntegerConfigKey("testCluster.maxSize", "Size after which it will throw InsufficientCapacityException", Integer.MAX_VALUE);

List<Integer> getSizeHistory();

List<Integer> getDesiredSizeHistory();
}
@@ -18,22 +18,32 @@
*/
package org.apache.brooklyn.core.test.entity;

import java.util.ArrayList;
import java.util.Collections;
import java.util.List;

import org.apache.brooklyn.core.entity.trait.Startable;
import org.apache.brooklyn.entity.group.DynamicClusterImpl;
import org.apache.brooklyn.util.collections.QuorumCheck.QuorumChecks;

import com.google.common.collect.ImmutableList;

/**
* Mock cluster entity for testing.
*/
public class TestClusterImpl extends DynamicClusterImpl implements TestCluster {
private volatile int size;

private final List<Integer> desiredSizeHistory = Collections.synchronizedList(new ArrayList<Integer>());
private final List<Integer> sizeHistory = Collections.synchronizedList(new ArrayList<Integer>());

public TestClusterImpl() {
}

@Override
public void init() {
super.init();
sizeHistory.add(size);
size = getConfig(INITIAL_SIZE);
sensors().set(Startable.SERVICE_UP, true);
}
@@ -48,10 +58,34 @@ protected void initEnrichers() {

@Override
public Integer resize(Integer desiredSize) {
desiredSizeHistory.add(desiredSize);

if (desiredSize > size) {
if (size < getConfig(MAX_SIZE)) {
desiredSize = Math.min(desiredSize, getConfig(MAX_SIZE));
} else {
throw new InsufficientCapacityException("Simulating insufficient capacity (desiredSize="+desiredSize+"; maxSize="+getConfig(MAX_SIZE)+"; currentSize="+size+")");
}
}
this.sizeHistory.add(desiredSize);
this.size = desiredSize;
return size;
}

@Override
public List<Integer> getSizeHistory() {
synchronized (sizeHistory) {
return ImmutableList.copyOf(sizeHistory);
}
}

@Override
public List<Integer> getDesiredSizeHistory() {
synchronized (desiredSizeHistory) {
return ImmutableList.copyOf(desiredSizeHistory);
}
}

@Override
public void stop() {
size = 0;
@@ -44,6 +44,7 @@
import org.apache.brooklyn.api.entity.Entity;
import org.apache.brooklyn.api.entity.EntitySpec;
import org.apache.brooklyn.api.location.Location;
import org.apache.brooklyn.api.location.NoMachinesAvailableException;
import org.apache.brooklyn.api.mgmt.Task;
import org.apache.brooklyn.api.sensor.SensorEvent;
import org.apache.brooklyn.core.entity.Attributes;
@@ -54,6 +55,7 @@
import org.apache.brooklyn.core.entity.lifecycle.ServiceStateLogic;
import org.apache.brooklyn.core.entity.trait.Changeable;
import org.apache.brooklyn.core.entity.trait.FailingEntity;
import org.apache.brooklyn.core.entity.trait.Resizable;
import org.apache.brooklyn.core.location.SimulatedLocation;
import org.apache.brooklyn.core.mgmt.BrooklynTaskTags;
import org.apache.brooklyn.core.test.BrooklynAppUnitTestSupport;
@@ -72,13 +74,13 @@
import org.testng.annotations.Test;

import com.google.common.base.Function;
import com.google.common.base.Predicate;
import com.google.common.base.Predicates;
import com.google.common.base.Throwables;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Iterables;
import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
import com.google.common.util.concurrent.Atomics;


@@ -202,6 +204,53 @@ public void resizeFromZeroToOneStartsANewEntityAndSetsItsParent() throws Excepti
assertEquals(entity.getApplication(), app);
}

@Test
public void testResizeWhereChildThrowsNoMachineAvailableExceptionIsPropagatedAsInsufficientCapacityException() throws Exception {
final DynamicCluster cluster = app.createAndManageChild(EntitySpec.create(DynamicCluster.class)
.configure(DynamicCluster.MEMBER_SPEC, EntitySpec.create(FailingEntity.class)
.configure(FailingEntity.FAIL_ON_START, true)
.configure(FailingEntity.EXCEPTION_CLAZZ, NoMachinesAvailableException.class))
.configure(DynamicCluster.INITIAL_SIZE, 0));
cluster.start(ImmutableList.of(loc));

try {
cluster.resize(1);
Asserts.shouldHaveFailedPreviously();
} catch (Exception e) {
Asserts.expectedFailureOfType(e, Resizable.InsufficientCapacityException.class);
}
}

@Test
public void testResizeWhereSubsetOfChildrenThrowsNoMachineAvailableExceptionReturnsNormally() throws Exception {
final DynamicCluster cluster = app.createAndManageChild(EntitySpec.create(DynamicCluster.class)
.configure(DynamicCluster.MEMBER_SPEC, EntitySpec.create(FailingEntity.class)
.configure(FailingEntity.FAIL_ON_START_CONDITION, new Predicate<FailingEntity>() {
final AtomicInteger counter = new AtomicInteger();
@Override public boolean apply(FailingEntity input) {
// Only second and subsequent entities fail
int index = counter.getAndIncrement();
return (index >= 1);
}})
.configure(FailingEntity.EXCEPTION_CLAZZ, NoMachinesAvailableException.class))
.configure(DynamicCluster.INITIAL_SIZE, 0));
cluster.start(ImmutableList.of(loc));

// Managed to partially resize, so should not fail entirely.
// Instead just say how big we managed to get.
Integer newSize = cluster.resize(2);
assertEquals(newSize, (Integer)1);
assertEquals(cluster.getCurrentSize(), (Integer)1);

// This attempt will fail, because all new children will fail
try {
cluster.resize(2);
Asserts.shouldHaveFailedPreviously();
} catch (Exception e) {
Asserts.expectedFailureOfType(e, Resizable.InsufficientCapacityException.class);
}
}

/** This can be sensitive to order, e.g. if TestEntity set expected RUNNING before setting SERVICE_UP,
* there would be a point when TestEntity is ON_FIRE.
* <p>
@@ -249,6 +298,7 @@ public void resizeDownByTwoAndDownByOne() throws Exception {
assertEquals(Iterables.size(Entities.descendants(cluster, TestEntity.class)), 0);
}


@Test
public void currentSizePropertyReflectsActualClusterSize() throws Exception {
DynamicCluster cluster = app.createAndManageChild(EntitySpec.create(DynamicCluster.class)
@@ -421,7 +471,7 @@ public void failingEntitiesDontBreakClusterActions() throws Exception {
}}));

cluster.start(ImmutableList.of(loc));
resizeExpectingError(cluster, 3);
cluster.resize(3);
assertEquals(cluster.getCurrentSize(), (Integer)2);
assertEquals(cluster.getMembers().size(), 2);
for (Entity member : cluster.getMembers()) {
@@ -533,7 +583,7 @@ public void testCanQuarantineFailedEntities() throws Exception {
}}));

cluster.start(ImmutableList.of(loc));
resizeExpectingError(cluster, 3);
cluster.resize(3);
assertEquals(cluster.getCurrentSize(), (Integer)2);
assertEquals(cluster.getMembers().size(), 2);
assertEquals(Iterables.size(Iterables.filter(cluster.getChildren(), Predicates.instanceOf(FailingEntity.class))), 3);
@@ -570,7 +620,7 @@ public void testDoNotQuarantineFailedEntities() throws Exception {
assertEquals(cluster.getChildren().size(), 0, "children="+cluster.getChildren());

// Failed node will not be a member or child
resizeExpectingError(cluster, 3);
cluster.resize(3);
assertEquals(cluster.getCurrentSize(), (Integer)2);
assertEquals(cluster.getMembers().size(), 2);
assertEquals(cluster.getChildren().size(), 2, "children="+cluster.getChildren());

0 comments on commit a91889d

Please sign in to comment.