Merge pull request #1593 from HubSpot/more_flaky_tests

More flaky tests

.travis.yml

@@ -5,9 +5,6 @@ jdk:
 install: true
 script: mvn -B -q verify
 
-git:
-  depth: 100
-
 sudo: false
 
 before_install:

SingularityBase/src/main/java/com/hubspot/singularity/SingularityTransformHelpers.java

@@ -64,7 +64,6 @@ public String apply(@Nonnull SingularityPendingDeploy input) {
   };
 
   public static final Function<SingularityKilledTaskIdRecord, String> KILLED_TASK_ID_RECORD_TO_REQUEST_ID = new Function<SingularityKilledTaskIdRecord, String>() {
-    @Nonnull
     @Override
     public String apply(@Nonnull SingularityKilledTaskIdRecord input) {
       return input.getTaskId().getRequestId();

SingularityService/pom.xml

@@ -378,6 +378,12 @@
       <scope>test</scope>
     </dependency>
 
+    <dependency>
+      <groupId>com.jayway.awaitility</groupId>
+      <artifactId>awaitility</artifactId>
+      <scope>test</scope>
+    </dependency>
+
     <dependency>
       <groupId>org.liquibase</groupId>
       <artifactId>liquibase-core</artifactId>

SingularityService/src/test/java/com/hubspot/singularity/SingularityCuratorTestBase.java

@@ -4,13 +4,18 @@
 import org.apache.curator.test.TestingServer;
 import org.junit.After;
 import org.junit.Before;
+import org.junit.Rule;
+import org.junit.rules.Timeout;
 
 import com.google.inject.Inject;
 import com.hubspot.singularity.scheduler.SingularityTestModule;
 import com.squarespace.jersey2.guice.JerseyGuiceUtils;
 
 public class SingularityCuratorTestBase {
 
+  @Rule
+  public Timeout globalTimeout = Timeout.seconds(30); // 30 seconds max for each @Test method
+
   @Inject
   protected CuratorFramework cf;
   @Inject

SingularityService/src/test/java/com/hubspot/singularity/SingularityHistoryTest.java

@@ -185,7 +185,7 @@ public void historyPurgerTest() {
 
     Assert.assertEquals(1, getTaskHistoryForRequest(requestId, 0, 10).size());
 
-    saveTasks(100, System.currentTimeMillis() - TimeUnit.DAYS.toMillis(200));
+    saveTasks(10, System.currentTimeMillis() - TimeUnit.DAYS.toMillis(200));
 
     purger.runActionOnPoll();
 

SingularityService/src/test/java/com/hubspot/singularity/scheduler/SingularityHealthchecksTest.java

@@ -24,6 +24,7 @@
 import com.hubspot.singularity.api.SingularityBounceRequest;
 import com.hubspot.singularity.api.SingularitySkipHealthchecksRequest;
 import com.hubspot.singularity.scheduler.SingularityNewTaskChecker.CheckTaskState;
+import com.jayway.awaitility.Awaitility;
 
 public class SingularityHealthchecksTest extends SingularitySchedulerTestBase {
 
@@ -368,7 +369,7 @@ public void testPortIndices() {
     try {
       setConfigurationForNoDelay();
       initRequest();
-      HealthcheckOptions options = new HealthcheckOptionsBuilder("http://uri").setPortIndex(Optional.of(1)).build();
+      HealthcheckOptions options = new HealthcheckOptionsBuilder("http://uri").setPortIndex(Optional.of(1)).setStartupDelaySeconds(Optional.of(0)).build();
       firstDeploy = initAndFinishDeploy(request, new SingularityDeployBuilder(request.getId(), firstDeployId).setCommand(Optional.of("sleep 100"))
           .setHealthcheck(Optional.of(options)), Optional.of(new Resources(1, 64, 3, 0)));
 
@@ -385,9 +386,7 @@ public void testPortIndices() {
 
       newTaskChecker.enqueueNewTaskCheck(firstTask, requestManager.getRequest(requestId), healthchecker);
 
-      finishNewTaskChecks();
-      finishHealthchecks();
-      finishNewTaskChecksAndCleanup();
+      Awaitility.await("healthcheck present").atMost(5, TimeUnit.SECONDS).until(() -> taskManager.getLastHealthcheck(firstTask.getTaskId()).isPresent());
 
       Assert.assertTrue(taskManager.getLastHealthcheck(firstTask.getTaskId()).get().toString().contains("host1:81"));
     } finally {
@@ -400,7 +399,7 @@ public void testPortNumber() {
     try {
       setConfigurationForNoDelay();
       initRequest();
-      HealthcheckOptions options = new HealthcheckOptionsBuilder("http://uri").setPortNumber(Optional.of(81L)).build();
+      HealthcheckOptions options = new HealthcheckOptionsBuilder("http://uri").setPortNumber(Optional.of(81L)).setStartupDelaySeconds(Optional.of(0)).build();
       firstDeploy = initAndFinishDeploy(request, new SingularityDeployBuilder(request.getId(), firstDeployId)
         .setCommand(Optional.of("sleep 100")).setResources(Optional.of(new Resources(1, 64, 3, 0)))
         .setHealthcheck(Optional.of(options)), Optional.absent());
@@ -418,9 +417,7 @@ public void testPortNumber() {
 
       newTaskChecker.enqueueNewTaskCheck(firstTask, requestManager.getRequest(requestId), healthchecker);
 
-      finishNewTaskChecks();
-      finishHealthchecks();
-      finishNewTaskChecksAndCleanup();
+      Awaitility.await("healthcheck present").atMost(5, TimeUnit.SECONDS).until(() -> taskManager.getLastHealthcheck(firstTask.getTaskId()).isPresent());
 
       Assert.assertTrue(taskManager.getLastHealthcheck(firstTask.getTaskId()).get().toString().contains("host1:81"));
     } finally {

SingularityService/src/test/java/com/hubspot/singularity/scheduler/SingularitySchedulerTest.java

@@ -90,6 +90,7 @@
 import com.hubspot.singularity.mesos.SingularityMesosTaskPrioritizer;
 import com.hubspot.singularity.scheduler.SingularityDeployHealthHelper.DeployHealth;
 import com.hubspot.singularity.scheduler.SingularityTaskReconciliation.ReconciliationState;
+import com.jayway.awaitility.Awaitility;
 
 public class SingularitySchedulerTest extends SingularitySchedulerTestBase {
   @Inject
@@ -524,15 +525,15 @@ public void testSchedulerRandomizesOffers() {
 
     sms.resourceOffers(driver, Arrays.asList(createOffer(20, 1024), createOffer(20, 1024)));
 
-    Assert.assertTrue(taskManager.getActiveTaskIds().size() == 15);
+    Assert.assertEquals(15, taskManager.getActiveTaskIds().size());
 
     Set<String> offerIds = Sets.newHashSet();
 
     for (SingularityTask activeTask : taskManager.getActiveTasks()) {
       offerIds.addAll(activeTask.getOffers().stream().map((o) -> o.getId().getValue()).collect(Collectors.toList()));
     }
 
-    Assert.assertTrue(offerIds.size() == 2);
+    Assert.assertEquals(2, offerIds.size());
   }
 
   @Test
@@ -1027,8 +1028,7 @@ public void testReconciliation() {
     initFirstDeploy();
 
     Assert.assertTrue(taskReconciliation.startReconciliation() == ReconciliationState.STARTED);
-    sleep(50);
-    Assert.assertTrue(!taskReconciliation.isReconciliationRunning());
+    Awaitility.await().atMost(10, TimeUnit.SECONDS).until(() -> !taskReconciliation.isReconciliationRunning());
 
     SingularityTask taskOne = launchTask(request, firstDeploy, 1, TaskState.TASK_STARTING);
     SingularityTask taskTwo = launchTask(request, firstDeploy, 2, TaskState.TASK_RUNNING);
@@ -1038,19 +1038,15 @@ public void testReconciliation() {
     Assert.assertTrue(taskReconciliation.startReconciliation() == ReconciliationState.STARTED);
     Assert.assertTrue(taskReconciliation.startReconciliation() == ReconciliationState.ALREADY_RUNNING);
 
-    sleep(50);
-    Assert.assertTrue(taskReconciliation.isReconciliationRunning());
+    Awaitility.await().atMost(10, TimeUnit.SECONDS).until(() -> taskReconciliation.isReconciliationRunning());
 
     saveLastActiveTaskStatus(taskOne, Optional.of(buildTaskStatus(taskOne)), +1000);
 
-    sleep(50);
-    Assert.assertTrue(taskReconciliation.isReconciliationRunning());
+    Awaitility.await().atMost(10, TimeUnit.SECONDS).until(() -> taskReconciliation.isReconciliationRunning());
 
     saveLastActiveTaskStatus(taskTwo, Optional.of(buildTaskStatus(taskTwo)), +1000);
 
-    sleep(50);
-
-    Assert.assertTrue(!taskReconciliation.isReconciliationRunning());
+    Awaitility.await().atMost(10, TimeUnit.SECONDS).until(() -> !taskReconciliation.isReconciliationRunning());
   }
 
 

SingularityService/src/test/java/com/hubspot/singularity/scheduler/SingularitySchedulerTestBase.java

@@ -12,6 +12,7 @@
 import java.util.concurrent.CancellationException;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.Future;
+import java.util.concurrent.ScheduledFuture;
 import java.util.concurrent.TimeUnit;
 
 import org.apache.mesos.Protos.Attribute;
@@ -387,7 +388,7 @@ protected void finishNewTaskChecksAndCleanup() {
   }
 
   protected void finishHealthchecks() {
-    for (Future<?> future : healthchecker.getHealthCheckFutures()) {
+    for (ScheduledFuture<?> future : healthchecker.getHealthCheckFutures()) {
       try {
         future.get();
       } catch (CancellationException ce) {

pom.xml

@@ -247,6 +247,14 @@
         <version>0.2.2</version>
       </dependency>
 
+      <dependency>
+        <groupId>com.jayway.awaitility</groupId>
+        <artifactId>awaitility</artifactId>
+        <version>1.7.0</version>
+        <scope>test</scope>
+      </dependency>
+
+
       <dependency>
         <groupId>com.amazonaws</groupId>
         <artifactId>aws-java-sdk-s3</artifactId>