From 275e114aae3179494eafff22f3c476370e7caac7 Mon Sep 17 00:00:00 2001 From: Tim Freeman Date: Mon, 13 Jun 2011 23:39:42 -0500 Subject: [PATCH] Tests for termination backoff suite --- scripts/integration-suites.sh | 2 +- .../xen/xenssh/MockShutdownTrash.java | 7 +- .../java/tests/suites/build.properties | 1 + service/service/java/tests/suites/build.xml | 8 +- .../tests/suites/failure/BackoffSuite.xml | 9 + .../nimbus/workspace-service/other/main.xml | 1 + .../testing/suites/failure/BackoffSuite.java | 291 ++++++++++++++++++ 7 files changed, 316 insertions(+), 3 deletions(-) create mode 100644 service/service/java/tests/suites/failure/BackoffSuite.xml create mode 100644 service/service/java/tests/suites/failure/src/org/globus/workspace/testing/suites/failure/BackoffSuite.java diff --git a/scripts/integration-suites.sh b/scripts/integration-suites.sh index adfa5db4..6abab876 100755 --- a/scripts/integration-suites.sh +++ b/scripts/integration-suites.sh @@ -1,7 +1,7 @@ #!/bin/bash -ALL_TEST_SUITES="basic01 basic02 basic03 basic04 basic05 basic06 failure01 spot01 spot02 spot03 spot04 spot05 spot06" +ALL_TEST_SUITES="basic01 basic02 basic03 basic04 basic05 basic06 failure01 failure02 spot01 spot02 spot03 spot04 spot05 spot06" if [ "X" == "X$1" ]; then diff --git a/service/service/java/source/src/org/globus/workspace/xen/xenssh/MockShutdownTrash.java b/service/service/java/source/src/org/globus/workspace/xen/xenssh/MockShutdownTrash.java index b47d5e7d..6e9366c8 100644 --- a/service/service/java/source/src/org/globus/workspace/xen/xenssh/MockShutdownTrash.java +++ b/service/service/java/source/src/org/globus/workspace/xen/xenssh/MockShutdownTrash.java @@ -35,6 +35,7 @@ public class MockShutdownTrash extends XenTask { private static boolean fail = false; private static int failCount = 0; + private static long msAtLastAttempt = 0; // Point of control from tests, ANY created task object will respect this static field // when the init() method is called -- so creating a number of instances simultaneously @@ -45,9 +46,13 @@ public static void setFail(boolean doFail) { public static int getFailCount() { return failCount; } + public static long getMsAtLastAttempt() { + return msAtLastAttempt; + } public static void resetFailCount() { failCount = 0; + msAtLastAttempt = 0; } protected void init() throws WorkspaceException { @@ -69,7 +74,7 @@ protected void init() throws WorkspaceException { logger.warn(this.name + " forced to fail."); failCount += 1; } - + msAtLastAttempt = System.currentTimeMillis(); this.cmd = (String[]) ssh.toArray(new String[ssh.size()]); } } diff --git a/service/service/java/tests/suites/build.properties b/service/service/java/tests/suites/build.properties index ab54fd82..478ae309 100644 --- a/service/service/java/tests/suites/build.properties +++ b/service/service/java/tests/suites/build.properties @@ -32,6 +32,7 @@ st.basic06=ParallelIdempotentCreationSuite # FAILURE SUITES st.failure.dir=${nimbus.suitesdir}/failure st.failure01=TerminateSuite +st.failure02=BackoffSuite # SPOT INSTANCE SUITES st.spot.dir=${nimbus.suitesdir}/spotinstances diff --git a/service/service/java/tests/suites/build.xml b/service/service/java/tests/suites/build.xml index 28555de3..f6520bab 100644 --- a/service/service/java/tests/suites/build.xml +++ b/service/service/java/tests/suites/build.xml @@ -215,8 +215,14 @@ + + + + + + - + diff --git a/service/service/java/tests/suites/failure/BackoffSuite.xml b/service/service/java/tests/suites/failure/BackoffSuite.xml new file mode 100644 index 00000000..df98a575 --- /dev/null +++ b/service/service/java/tests/suites/failure/BackoffSuite.xml @@ -0,0 +1,9 @@ + + + + + + + + + diff --git a/service/service/java/tests/suites/failure/home/services/etc/nimbus/workspace-service/other/main.xml b/service/service/java/tests/suites/failure/home/services/etc/nimbus/workspace-service/other/main.xml index a76ff1a3..41e15b7b 100644 --- a/service/service/java/tests/suites/failure/home/services/etc/nimbus/workspace-service/other/main.xml +++ b/service/service/java/tests/suites/failure/home/services/etc/nimbus/workspace-service/other/main.xml @@ -339,6 +339,7 @@ + diff --git a/service/service/java/tests/suites/failure/src/org/globus/workspace/testing/suites/failure/BackoffSuite.java b/service/service/java/tests/suites/failure/src/org/globus/workspace/testing/suites/failure/BackoffSuite.java new file mode 100644 index 00000000..769efc4d --- /dev/null +++ b/service/service/java/tests/suites/failure/src/org/globus/workspace/testing/suites/failure/BackoffSuite.java @@ -0,0 +1,291 @@ +/* + * Copyright 1999-2011 University of Chicago + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.globus.workspace.testing.suites.failure; + +import static org.testng.AssertJUnit.assertFalse; +import static org.testng.AssertJUnit.assertEquals; +import static org.testng.AssertJUnit.assertNotNull; +import static org.testng.AssertJUnit.assertTrue; + +import org.globus.workspace.testing.NimbusTestBase; +import org.globus.workspace.testing.NimbusTestContextLoader; +import org.globus.workspace.xen.xenssh.MockShutdownTrash; +import org.nimbustools.api.repr.Caller; +import org.nimbustools.api.repr.CreateResult; +import org.nimbustools.api.repr.vm.VM; +import org.nimbustools.api.services.rm.Manager; +import org.springframework.test.context.ContextConfiguration; +import org.testng.annotations.AfterSuite; +import org.testng.annotations.Test; + + +@ContextConfiguration( + locations={"file:./service/service/java/tests/suites/failure/home/services/etc/nimbus/workspace-service/other/main.xml"}, + loader=NimbusTestContextLoader.class) +public class BackoffSuite extends NimbusTestBase { + + // ----------------------------------------------------------------------------------------- + // extends NimbusTestBase + // ----------------------------------------------------------------------------------------- + + @AfterSuite(alwaysRun=true) + @Override + public void suiteTeardown() throws Exception { + super.suiteTeardown(); + } + + /** + * This is how coordinate your Java test suite code with the conf files to use. + * @return absolute path to the value that should be set for $NIMBUS_HOME + * @throws Exception if $NIMBUS_HOME cannot be determined + */ + @Override + protected String getNimbusHome() throws Exception { + return this.determineSuitesPath() + "/failure/home"; + } + + + // ----------------------------------------------------------------------------------------- + // PREREQ TESTS (if any of these fail, nothing else will work at all) + // ----------------------------------------------------------------------------------------- + + /** + * Check if ModuleLocator can be retrieved and used at all. + */ + @Test(groups="pr3r3qs") + public void retrieveModuleLocator() throws Exception { + logger.debug("retrieveModuleLocator"); + final Manager rm = this.locator.getManager(); + final VM[] vms = rm.getGlobalAll(); + + // we know there are zero so far because it is in group 'prereqs' + assertEquals(0, vms.length); + } + + + // ----------------------------------------------------------------------------------------- + // FAILURE BEHAVIOR + // ----------------------------------------------------------------------------------------- + + @Test(groups="nobackoff", dependsOnGroups="pr3r3qs") + public void noBackoff() throws Exception { + logger.debug("nobackoff"); + final Manager rm = this.locator.getManager(); + + VM[] allvms = rm.getGlobalAll(); + assertEquals(0, allvms.length); + + MockShutdownTrash.resetFailCount(); + + final Caller caller = this.populator().getCaller(); + final CreateResult result = + rm.create(this.populator().getCreateRequest("suite:backoff:nobackoff"), + caller); + + final VM[] vms = result.getVMs(); + assertEquals(1, vms.length); + assertNotNull(vms[0]); + logger.info("Leased vm '" + vms[0].getID() + '\''); + + assertTrue(rm.exists(vms[0].getID(), Manager.INSTANCE)); + + Thread.sleep(2000L); + assertEquals(0, MockShutdownTrash.getFailCount()); + + // Fail at killing the instance: + MockShutdownTrash.setFail(true); + logger.warn("Set to fail."); + + rm.trash(vms[0].getID(), Manager.INSTANCE, caller); + Thread.sleep(2000L); + + // Check that it is at least trying to terminate the node: + int lastFailCount = MockShutdownTrash.getFailCount(); + assertFalse(0 == lastFailCount); + + assertTrue(rm.exists(vms[0].getID(), Manager.INSTANCE)); + allvms = rm.getGlobalAll(); + assertEquals(1, allvms.length); + + long lastFailMs = MockShutdownTrash.getMsAtLastAttempt(); + assertFalse(0 == lastFailMs); + + Thread.sleep(4000L); + + long nextFailMs = MockShutdownTrash.getMsAtLastAttempt(); + assertFalse(nextFailMs == lastFailMs); + + assertTrue(rm.exists(vms[0].getID(), Manager.INSTANCE)); + allvms = rm.getGlobalAll(); + assertEquals(1, allvms.length); + + // Start succeeding to kill the instance: + MockShutdownTrash.setFail(false); + logger.warn("Set to succeed."); + + Thread.sleep(4000L); + + // Should now be gone: + assertFalse(rm.exists(vms[0].getID(), Manager.INSTANCE)); + allvms = rm.getGlobalAll(); + assertEquals(0, allvms.length); + + long finalFailMs = MockShutdownTrash.getMsAtLastAttempt(); + assertFalse(finalFailMs == nextFailMs); + + // It should have stopped + Thread.sleep(4000L); + assertEquals(finalFailMs, MockShutdownTrash.getMsAtLastAttempt()); + } + + + @Test(groups="yesbackoff", dependsOnGroups="nobackoff") + public void testBackoff() throws Exception { + logger.debug("yesbackoff"); + final Manager rm = this.locator.getManager(); + + VM[] allvms = rm.getGlobalAll(); + assertEquals(0, allvms.length); + + MockShutdownTrash.resetFailCount(); + + final Caller caller = this.populator().getCaller(); + final CreateResult result = + rm.create(this.populator().getCreateRequest("suite:backoff:yesbackoff"), + caller); + + final VM[] vms = result.getVMs(); + assertEquals(1, vms.length); + assertNotNull(vms[0]); + logger.info("Leased vm '" + vms[0].getID() + '\''); + + assertTrue(rm.exists(vms[0].getID(), Manager.INSTANCE)); + + Thread.sleep(2000L); + assertEquals(0, MockShutdownTrash.getFailCount()); + + // Fail at killing the instance: + MockShutdownTrash.setFail(true); + logger.warn("Set to fail."); + + rm.trash(vms[0].getID(), Manager.INSTANCE, caller); + Thread.sleep(2000L); + + // Check that it is at least trying to terminate the node: + int lastFailCount = MockShutdownTrash.getFailCount(); + assertFalse(0 == lastFailCount); + + assertTrue(rm.exists(vms[0].getID(), Manager.INSTANCE)); + allvms = rm.getGlobalAll(); + assertEquals(1, allvms.length); + + long prevFailMs = MockShutdownTrash.getMsAtLastAttempt(); + assertFalse(0 == prevFailMs); + + Thread.sleep(4000L); + + long nextFailMs = MockShutdownTrash.getMsAtLastAttempt(); + assertFalse(nextFailMs == prevFailMs); + + logger.debug("fail difference: " + (nextFailMs - prevFailMs)); + + prevFailMs = nextFailMs; + nextFailMs = 0; + + assertTrue(rm.exists(vms[0].getID(), Manager.INSTANCE)); + allvms = rm.getGlobalAll(); + assertEquals(1, allvms.length); + + long markerTime1 = prevFailMs; + + Thread.sleep(2100L); + nextFailMs = MockShutdownTrash.getMsAtLastAttempt(); + logger.debug("fail difference: " + (nextFailMs - prevFailMs)); + prevFailMs = nextFailMs; + nextFailMs = 0; + + Thread.sleep(2300L); + nextFailMs = MockShutdownTrash.getMsAtLastAttempt(); + logger.debug("fail difference: " + (nextFailMs - prevFailMs)); + prevFailMs = nextFailMs; + nextFailMs = 0; + + Thread.sleep(1900L); + nextFailMs = MockShutdownTrash.getMsAtLastAttempt(); + logger.debug("fail difference: " + (nextFailMs - prevFailMs)); + prevFailMs = nextFailMs; + nextFailMs = 0; + + Thread.sleep(2100L); + + assertEquals(5, MockShutdownTrash.getFailCount()); + + Thread.sleep(5000L); + nextFailMs = MockShutdownTrash.getMsAtLastAttempt(); + logger.debug("fail difference: " + (nextFailMs - prevFailMs)); + prevFailMs = nextFailMs; + nextFailMs = 0; + + Thread.sleep(4100L); + + assertEquals(6, MockShutdownTrash.getFailCount()); + + nextFailMs = MockShutdownTrash.getMsAtLastAttempt(); + logger.debug("fail difference: " + (nextFailMs - prevFailMs)); + prevFailMs = nextFailMs; + nextFailMs = 0; + + Thread.sleep(4300L); + nextFailMs = MockShutdownTrash.getMsAtLastAttempt(); + logger.debug("fail difference: " + (nextFailMs - prevFailMs)); + prevFailMs = nextFailMs; + nextFailMs = 0; + + Thread.sleep(4900L); + nextFailMs = MockShutdownTrash.getMsAtLastAttempt(); + logger.debug("fail difference: " + (nextFailMs - prevFailMs)); + prevFailMs = nextFailMs; + nextFailMs = 0; + + Thread.sleep(3100L); + nextFailMs = MockShutdownTrash.getMsAtLastAttempt(); + logger.debug("fail difference: " + (nextFailMs - prevFailMs)); + prevFailMs = nextFailMs; + nextFailMs = 0; + + Thread.sleep(4300L); + nextFailMs = MockShutdownTrash.getMsAtLastAttempt(); + logger.debug("fail difference: " + (nextFailMs - prevFailMs)); + prevFailMs = nextFailMs; + nextFailMs = 0; + + assertEquals(7, MockShutdownTrash.getFailCount()); + + Thread.sleep(5100L); + nextFailMs = MockShutdownTrash.getMsAtLastAttempt(); + logger.debug("fail difference: " + (nextFailMs - prevFailMs)); + prevFailMs = nextFailMs; + nextFailMs = 0; + + Thread.sleep(6300L); + nextFailMs = MockShutdownTrash.getMsAtLastAttempt(); + logger.debug("fail difference: " + (nextFailMs - prevFailMs)); + + assertEquals(8, MockShutdownTrash.getFailCount()); + } + +}