Skip to content

Commit

Permalink
Added a maintenance test case with live alerts
Browse files Browse the repository at this point in the history
  • Loading branch information
pnguyen committed Nov 6, 2009
1 parent fe1743b commit 9464534
Show file tree
Hide file tree
Showing 7 changed files with 257 additions and 71 deletions.
25 changes: 14 additions & 11 deletions src/org/hyperic/hq/hqapi1/test/AlertDefinitionTestBase.java
View file
Expand Up @@ -94,17 +94,19 @@ protected List<AlertDefinition> syncAlertDefinitions(List<AlertDefinition> defin
return response.getAlertDefinition();
}

protected AlertDefinition createProblemAlertDefinition(Resource resource,
Escalation e,
boolean isResourceType,
boolean willRecover)
protected AlertDefinition createAvailabilityAlertDefinition(Resource resource,
Escalation e,
boolean isResourceType,
boolean willRecover,
double availability)
throws IOException {

// Find availability metric for the passed in resource
Metric availMetric = findAvailabilityMetric(resource);

// Create alert definition
String name = "Test" + (isResourceType ? " Resource Type " : " ") + "Problem Alert";
String name = "Test" + (isResourceType ? " Resource Type " : " ")
+ "Availability=" + availability + " Alert";
AlertDefinition d = generateTestDefinition(name);
d.setWillRecover(willRecover);
if (isResourceType) {
Expand All @@ -118,17 +120,18 @@ protected AlertDefinition createProblemAlertDefinition(Resource resource,
AlertCondition threshold =
AlertDefinitionBuilder.createThresholdCondition(
true, availMetric.getName(),
AlertDefinitionBuilder.AlertComparator.EQUALS, 0);
AlertDefinitionBuilder.AlertComparator.EQUALS,
availability);
d.getAlertCondition().add(threshold);
AlertDefinition newDef = syncAlertDefinition(d);

if (isResourceType) {
validateTypeDefinition(newDef);
}

validateProblemAlertDefinition(newDef, willRecover, true);
validateAvailabilityAlertDefinition(newDef, willRecover, true);

assertTrue("The problem alert definition should have "
assertTrue("The alert definition should have "
+ ((e == null) ? "no" : "an") + " escalation",
(e == null) ? newDef.getEscalation() == null
: newDef.getEscalation() != null);
Expand Down Expand Up @@ -171,9 +174,9 @@ protected void validateTypeDefinition(AlertDefinition d) {
d.getResourcePrototype() != null);
}

protected void validateProblemAlertDefinition(AlertDefinition def,
boolean willRecover,
boolean enabled) {
protected void validateAvailabilityAlertDefinition(AlertDefinition def,
boolean willRecover,
boolean enabled) {
assertTrue("The problem alert definition should be active",
def.isActive());
assertTrue("The problem alert definition's willRecover flag should be " + willRecover,
Expand Down
43 changes: 13 additions & 30 deletions src/org/hyperic/hq/hqapi1/test/AlertFireRecovery_test.java
View file
Expand Up @@ -115,10 +115,12 @@ public void testFireProblemAlertAndThenCreateAndFireRecoveryAlert()
Resource platform = getLocalPlatformResource(false, false);
boolean willRecover = false;
AlertDefinition problemDef = createProblemAlertDefinition(platform, null, false, willRecover);
Alert problemAlert = fireProblemAlert(problemDef, willRecover);
AlertDefinition problemDef =
createAvailabilityAlertDefinition(platform, null, false, willRecover, 0);
Alert problemAlert = fireAvailabilityAlert(problemDef, willRecover, 0);
AlertDefinition recoveryDef = createRecoveryAlertDefinition(platform, problemDef, false);
AlertDefinition recoveryDef =
createRecoveryAlertDefinition(platform, problemDef, false);
fireRecoveryAlert(recoveryDef, problemAlert, willRecover);
// Cleanup
Expand All @@ -138,7 +140,7 @@ private void createAndFireAlerts(Escalation escalation,
Resource platform = getLocalPlatformResource(false, false);

AlertDefinition problemDef =
createProblemAlertDefinition(platform, escalation, isResourceType, willRecover);
createAvailabilityAlertDefinition(platform, escalation, isResourceType, willRecover, 0);
AlertDefinition recoveryDef =
createRecoveryAlertDefinition(platform, problemDef, addRecoveryPostCreate);

Expand All @@ -149,7 +151,7 @@ private void createAndFireAlerts(Escalation escalation,
problemDefToFire = problemDef;
}

Alert problemAlert = fireProblemAlert(problemDefToFire, willRecover);
Alert problemAlert = fireAvailabilityAlert(problemDefToFire, willRecover, 0);

AlertDefinition recoveryDefToFire = null;
if (isResourceType) {
Expand All @@ -167,29 +169,6 @@ private void createAndFireAlerts(Escalation escalation,
cleanup(definitions);
}

private Alert fireProblemAlert(AlertDefinition problemDef,
boolean willRecover)
throws Exception {

long start = System.currentTimeMillis();

// Insert a fake 'down' measurement so that
// the problem alert definition will fire.
sendAvailabilityDataPoint(problemDef.getResource(), 0.0);

Alert problemAlert = findAlert(problemDef, start);
assertFalse("The problem alert should not be fixed",
problemAlert.isFixed());

// Get the updated problem alert definition
AlertDefinition updatedDef = getAlertDefinition(problemAlert.getAlertDefinitionId());
validateProblemAlertDefinition(updatedDef,
willRecover,
willRecover ? false : true);

return problemAlert;
}

private Alert fireRecoveryAlert(AlertDefinition recoveryDef,
Alert problemAlert,
boolean willRecover)
Expand All @@ -204,15 +183,19 @@ private Alert fireRecoveryAlert(AlertDefinition recoveryDef,
Alert recoveryAlert = findAlert(recoveryDef, start);
assertTrue("The recovery alert should be fixed",
recoveryAlert.isFixed());


// Get the updated recovery alert definition
AlertDefinition updatedRecoveryDef = getAlertDefinition(recoveryDef.getId());
validateAvailabilityAlertDefinition(updatedRecoveryDef, false, true);

// Get the updated problem alert
problemAlert = getAlert(problemAlert.getId());
assertTrue("The problem alert should be fixed",
problemAlert.isFixed());

// Get the updated problem alert definition
AlertDefinition problemDef = getAlertDefinition(problemAlert.getAlertDefinitionId());
validateProblemAlertDefinition(problemDef, willRecover, true);
validateAvailabilityAlertDefinition(problemDef, willRecover, true);

return recoveryAlert;
}
Expand Down
24 changes: 24 additions & 0 deletions src/org/hyperic/hq/hqapi1/test/AlertTestBase.java
View file
Expand Up @@ -117,6 +117,30 @@ protected Alert generateAlerts(Resource resource,
return findAlert(def, start);
}

protected Alert fireAvailabilityAlert(AlertDefinition def,
boolean willRecover,
double availability)
throws Exception {

long start = System.currentTimeMillis();

// Insert a fake availability data point so that
// the alert definition will fire.
sendAvailabilityDataPoint(def.getResource(), availability);

Alert alert = findAlert(def, start);
assertFalse("The alert should not be fixed",
alert.isFixed());

// Get the updated alert definition
AlertDefinition updatedDef = getAlertDefinition(alert.getAlertDefinitionId());
validateAvailabilityAlertDefinition(updatedDef,
willRecover,
willRecover ? false : true);

return alert;
}

protected void sendAvailabilityDataPoint(Resource resource, double availability)
throws IOException {

Expand Down
5 changes: 3 additions & 2 deletions src/org/hyperic/hq/hqapi1/test/HierarchicalAlertingTestBase.java
View file
Expand Up @@ -27,7 +27,7 @@ protected void simulatePlatformDownServersDown(boolean enabled)

Resource platform = getLocalPlatformResource(false, true);
AlertDefinition platformAlertDef =
createProblemAlertDefinition(platform, null, false, true);
createAvailabilityAlertDefinition(platform, null, false, true, 0);

List<AlertDefinition> problemAlertDefs = new ArrayList<AlertDefinition>();
for (Resource server : platform.getResource()) {
Expand All @@ -39,7 +39,8 @@ protected void simulatePlatformDownServersDown(boolean enabled)

// set alert definition to willRecover=true
// so that it will fire only once
problemAlertDefs.add(createProblemAlertDefinition(server, null, false, true));
problemAlertDefs.add(
createAvailabilityAlertDefinition(server, null, false, true, 0));
}
}

Expand Down
13 changes: 4 additions & 9 deletions src/org/hyperic/hq/hqapi1/test/MaintenanceGet_test.java
View file
Expand Up @@ -72,15 +72,10 @@ public void testGet() throws Exception {
Group g = getFileServerMountCompatibleGroup();
long start = System.currentTimeMillis() + HOUR;
long end = start + HOUR;
MaintenanceResponse scheduleResponse = mApi.schedule(g.getId(),
start, end);
hqAssertSuccess(scheduleResponse);

MaintenanceResponse getResponse = mApi.get(g.getId());
hqAssertSuccess(getResponse);

MaintenanceEvent e = getResponse.getMaintenanceEvent();
assertNotNull("MaintenanceEvent not found for valid group " + g.getName());

MaintenanceEvent e = schedule(g, start, end);
e = get(g);
assertNotNull("Maintenance event not found for valid group " + g.getName(), e);
valididateMaintenanceEvent(e, g, start, end);

StatusResponse unscheduleResponse = mApi.unschedule(g.getId());
Expand Down
120 changes: 113 additions & 7 deletions src/org/hyperic/hq/hqapi1/test/MaintenanceSchedule_test.java
View file
Expand Up @@ -27,22 +27,32 @@

package org.hyperic.hq.hqapi1.test;

import org.hyperic.hq.hqapi1.GroupApi;
import org.hyperic.hq.hqapi1.HQApi;
import org.hyperic.hq.hqapi1.MaintenanceApi;
import org.hyperic.hq.hqapi1.types.Alert;
import org.hyperic.hq.hqapi1.types.AlertDefinition;
import org.hyperic.hq.hqapi1.types.MaintenanceEvent;
import org.hyperic.hq.hqapi1.types.MaintenanceResponse;
import org.hyperic.hq.hqapi1.types.MaintenanceState;
import org.hyperic.hq.hqapi1.types.Group;
import org.hyperic.hq.hqapi1.types.GroupResponse;
import org.hyperic.hq.hqapi1.types.Operation;
import org.hyperic.hq.hqapi1.types.Resource;
import org.hyperic.hq.hqapi1.types.Role;
import org.hyperic.hq.hqapi1.types.RoleResponse;
import org.hyperic.hq.hqapi1.types.StatusResponse;
import org.hyperic.hq.hqapi1.types.User;

import java.util.Collections;
import java.util.List;
import java.util.Random;

public class MaintenanceSchedule_test extends MaintenanceTestBase {

private static final long HOUR = 60 * 60 * 1000;
private static final long SECOND = 1000;
private static final long MINUTE = 60 * SECOND;
private static final long HOUR = 60 * MINUTE;

public MaintenanceSchedule_test(String name) {
super(name);
Expand Down Expand Up @@ -92,19 +102,79 @@ public void testSchedule() throws Exception {
Group g = getFileServerMountCompatibleGroup();
long start = System.currentTimeMillis() + HOUR;
long end = start + HOUR;
MaintenanceResponse response = mApi.schedule(g.getId(),
start, end);
hqAssertSuccess(response);

assertNotNull(response.getMaintenanceEvent());
valididateMaintenanceEvent(response.getMaintenanceEvent(), g, start, end);

MaintenanceEvent event = schedule(g, start, end);

StatusResponse unscheduleResponse = mApi.unschedule(g.getId());
hqAssertSuccess(unscheduleResponse);

cleanupGroup(g);
}

public void testFireAlertsBeforeSchedulingCompatibleGroup()
throws Exception {

HQApi api = getApi();

// create resource
Resource resource = createControllableResource(api);

// create group
Group maintGroup = createGroup(Collections.singletonList(resource));

// create alert definitions
AlertDefinition alertDefFireOnce =
createAvailabilityAlertDefinition(resource, null, false, true, 1);
AlertDefinition alertDefFireEveryTime =
createAvailabilityAlertDefinition(resource, null, false, false, 1);

// insert a fake 'up' measurement so that
// the alert definitions will fire.
long alertStart = System.currentTimeMillis();
Alert alertFireOnce = fireAvailabilityAlert(alertDefFireOnce, true, 1);
Alert alertFireEveryTime = fireAvailabilityAlert(alertDefFireEveryTime, false, 1);

// TODO check measurement enabled status

// schedule maintenance
MaintenanceApi mApi = api.getMaintenanceApi();
long maintStart = System.currentTimeMillis() + 5*SECOND;
long maintEnd = maintStart + MINUTE;

MaintenanceEvent event = schedule(maintGroup, maintStart, maintEnd);

// wait for maintenance to start
waitForMaintenanceStateChange(maintGroup, MaintenanceState.RUNNING);

// validate alert definitions during the maintenance
// the internal enabled flag should be false for all alert definitions
alertDefFireOnce = getAlertDefinition(alertDefFireOnce.getId());
validateAvailabilityAlertDefinition(alertDefFireOnce, true, false);

alertDefFireEveryTime = getAlertDefinition(alertDefFireEveryTime.getId());
validateAvailabilityAlertDefinition(alertDefFireEveryTime, false, false);

// TODO check measurement enabled status

// wait for maintenance to end
waitForMaintenanceStateChange(maintGroup, MaintenanceState.COMPLETE);

// validate alert definitions after the maintenance
// the internal enabled flag should still be false for the willRecover
// alert definition that fired before the maintenance
alertDefFireOnce = getAlertDefinition(alertDefFireOnce.getId());
validateAvailabilityAlertDefinition(alertDefFireOnce, true, false);

// the internal enabled flag should be true after the maintenance for
// the alert definition that is configured to fire every time
alertDefFireEveryTime = getAlertDefinition(alertDefFireEveryTime.getId());
validateAvailabilityAlertDefinition(alertDefFireEveryTime, false, true);

// TODO check measurement enabled status

cleanupGroup(maintGroup, true);
}

public void testScheduleNoGroupPermission() throws Exception {

List<User> users = createTestUsers(1);
Expand Down Expand Up @@ -173,4 +243,40 @@ public void testScheduleNoMaintenancePermission() throws Exception {
cleanupRole(viewRole);
cleanupGroup(groupWithRole);
}

private void waitForMaintenanceStateChange(Group g, MaintenanceState newState)
throws Exception{

MaintenanceEvent event = get(g);
assertNotNull("The group must have a scheduled maintenance event",
event);
MaintenanceState initialState = event.getState();
MaintenanceState currentState = event.getState();

long timeout = 0;
if (newState.value().equals(MaintenanceState.RUNNING.value())) {
timeout = event.getStartTime() + 30*SECOND;
} else if (newState.value().equals(MaintenanceState.COMPLETE.value())) {
timeout = event.getEndTime() + 30*SECOND;
}

while (!currentState.value().equals(newState.value())) {
if (System.currentTimeMillis() >= timeout) {
String message = "The maintenance event did not change state from "
+ initialState.value() + " to "
+ newState.value() + " in time.";
throw new Exception(message);
}

pauseTest(5*SECOND);

event = get(g);

if (event == null) {
currentState = MaintenanceState.COMPLETE;
} else {
currentState = get(g).getState();
}
}
}
}

0 comments on commit 9464534

Please sign in to comment.