Add Timeouts for Staging and Starting Apps

Previously staging and starting apps had fixed wait periods of ~90 seconds, which were inadequate for real-world usage. This change adds the ability to set separate timeout values for each of these applicable operations requests (e.g. PushApplicationRequest), with defaults matching the CLI of 15 minutes (staging) and 5 minutes (startup). It also updates the process of waiting for jobs to use the same style, though currently without the ability to set a custom timeout on a request. [#115712029][#115711927]
cloudfoundry · Mar 22, 2016 · 1bbf9d1 · nebhale · Mar 22, 2016 · nebhale
1 parent 17c8ee0
commit 1bbf9d1
Show file tree

Hide file tree

Showing 11 changed files with 293 additions and 45 deletions.
diff --git a/...perations/src/main/java/org/cloudfoundry/operations/applications/DefaultApplications.java b/...perations/src/main/java/org/cloudfoundry/operations/applications/DefaultApplications.java
@@ -151,7 +151,8 @@ public Mono<Void> copySource(CopySourceApplicationRequest request) {
                         .after(() -> Mono.just(targetApplicationId))
                 )))
             .where(predicate((request1, targetApplicationId) -> Optional.ofNullable(request1.getRestart()).orElse(false)))
-            .then(function((request1, targetApplicationId) -> restartApplication(this.cloudFoundryClient, request1.getTargetName(), targetApplicationId)))
+            .then(function((request1, targetApplicationId) ->
+                restartApplication(this.cloudFoundryClient, request1.getTargetName(), targetApplicationId, request1.getStagingTimeout(), request1.getStartupTimeout())))
             .after();
     }
 
@@ -266,7 +267,8 @@ public Mono<Void> push(PushApplicationRequest request) {
             .as(thenKeep(function((applicationId, request1) -> uploadApplicationAndWait(this.cloudFoundryClient, applicationId, request1.getApplication()))))
             .as(thenKeep(function((applicationId, request1) -> stopApplication(this.cloudFoundryClient, applicationId))))
             .where(predicate((applicationId, request1) -> !Optional.ofNullable(request1.getNoStart()).orElse(false)))
-            .then(function((applicationId, request1) -> startApplicationAndWait(this.cloudFoundryClient, request1.getName(), applicationId)))
+            .then(function((applicationId, request1) -> startApplicationAndWait(this.cloudFoundryClient, request1.getName(), applicationId, request1.getStagingTimeout(), request1.getStartupTimeout
+                ())))
             .after();
     }
 
@@ -276,8 +278,8 @@ public Mono<Void> rename(RenameApplicationRequest request) {
             .when(ValidationUtils.validate(request), this.spaceId)
             .then(function((request1, spaceId) -> Mono
                 .when(
-                    getApplicationId(this.cloudFoundryClient, request.getName(), spaceId),
-                    Mono.just(request.getNewName())
+                    getApplicationId(this.cloudFoundryClient, request1.getName(), spaceId),
+                    Mono.just(request1.getNewName())
                 )))
             .then(function((applicationId, newName) -> requestUpdateApplicationRename(this.cloudFoundryClient, applicationId, newName)))
             .after();
@@ -290,7 +292,7 @@ public Mono<Void> restage(RestageApplicationRequest request) {
             .then(function((request1, spaceId) -> Mono
                 .just(request1)
                 .and(getApplicationId(this.cloudFoundryClient, request1.getName(), spaceId))))
-            .then(function((request1, applicationId) -> restageApplication(this.cloudFoundryClient, request1.getName(), applicationId)))
+            .then(function((request1, applicationId) -> restageApplication(this.cloudFoundryClient, request1.getName(), applicationId, request1.getStagingTimeout(), request1.getStartupTimeout())))
             .after();
     }
 
@@ -299,9 +301,9 @@ public Mono<Void> restart(RestartApplicationRequest request) {
         return Mono
             .when(ValidationUtils.validate(request), this.spaceId)
             .then(function((request1, spaceId) -> getApplication(this.cloudFoundryClient, request1.getName(), spaceId)
-                .and(Mono.just(request1.getName()))))
-            .then(function((resource, application) -> stopApplicationIfNotStopped(cloudFoundryClient, resource)
-                .then(resource1 -> startApplicationAndWait(this.cloudFoundryClient, application, ResourceUtils.getId(resource1)))))
+                .and(Mono.just(request1))))
+            .then(function((resource, request1) -> stopApplicationIfNotStopped(cloudFoundryClient, resource)
+                .then(resource1 -> startApplicationAndWait(this.cloudFoundryClient, request1.getName(), ResourceUtils.getId(resource1), request1.getStagingTimeout(), request1.getStartupTimeout()))))
             .after();
     }
 
@@ -337,7 +339,8 @@ public Mono<Void> scale(ScaleApplicationRequest request) {
                     requestUpdateApplicationScale(this.cloudFoundryClient, applicationId, request1.getDiskLimit(), request1.getInstances(), request1.getMemoryLimit())
                 )))
             .where(predicate(DefaultApplications::isRestartRequired))
-            .then(function((request1, resource) -> restartApplication(this.cloudFoundryClient, request1.getName(), ResourceUtils.getId(resource))))
+            .then(function((request1, resource) ->
+                restartApplication(this.cloudFoundryClient, request1.getName(), ResourceUtils.getId(resource), request1.getStagingTimeout(), request1.getStartupTimeout())))
             .after();
     }
 
@@ -368,8 +371,9 @@ public Mono<Void> start(StartApplicationRequest request) {
         return Mono
             .when(ValidationUtils.validate(request), this.spaceId)
             .then(function((request1, spaceId) -> getApplicationIdWhere(this.cloudFoundryClient, request1.getName(), spaceId, isNotIn(STARTED_STATE))
-                .and(Mono.just(request1.getName()))))
-            .then(function((applicationId, application) -> startApplicationAndWait(this.cloudFoundryClient, application, applicationId)))
+                .and(Mono.just(request1))))
+            .then(function((applicationId, request1) ->
+                startApplicationAndWait(this.cloudFoundryClient, request1.getName(), applicationId, request1.getStagingTimeout(), request1.getStartupTimeout())))
             .after();
     }
 
@@ -1079,25 +1083,25 @@ private static Mono<UploadApplicationResponse> requestUploadApplication(CloudFou
                 .build());
     }
 
-    private static Mono<String> restageApplication(CloudFoundryClient cloudFoundryClient, String application, String applicationId) {
+    private static Mono<String> restageApplication(CloudFoundryClient cloudFoundryClient, String application, String applicationId, Duration stagingTimeout, Duration startupTimeout) {
         return requestRestageApplication(cloudFoundryClient, applicationId)
-            .then(response -> waitForStaging(cloudFoundryClient, application, applicationId))
-            .then(state -> waitForRunning(cloudFoundryClient, application, applicationId));
+            .then(response -> waitForStaging(cloudFoundryClient, application, applicationId, stagingTimeout))
+            .then(state -> waitForRunning(cloudFoundryClient, application, applicationId, startupTimeout));
     }
 
-    private static Mono<String> restartApplication(CloudFoundryClient cloudFoundryClient, String application, String applicationId) {
+    private static Mono<String> restartApplication(CloudFoundryClient cloudFoundryClient, String application, String applicationId, Duration stagingTimeout, Duration startupTimeout) {
         return stopApplication(cloudFoundryClient, applicationId)
-            .then(abstractApplicationResource -> startApplicationAndWait(cloudFoundryClient, application, applicationId));
+            .then(abstractApplicationResource -> startApplicationAndWait(cloudFoundryClient, application, applicationId, stagingTimeout, startupTimeout));
     }
 
     private static Predicate<AbstractApplicationResource> sshEnabled(Boolean enabled) {
         return resource -> enabled.equals(ResourceUtils.getEntity(resource).getEnableSsh());
     }
 
-    private static Mono<String> startApplicationAndWait(CloudFoundryClient cloudFoundryClient, String application, String applicationId) {
+    private static Mono<String> startApplicationAndWait(CloudFoundryClient cloudFoundryClient, String application, String applicationId, Duration stagingTimeout, Duration startupTimeout) {
         return requestUpdateApplicationState(cloudFoundryClient, applicationId, STARTED_STATE)
-            .then(response -> waitForStaging(cloudFoundryClient, application, applicationId))
-            .then(state -> waitForRunning(cloudFoundryClient, application, applicationId));
+            .then(response -> waitForStaging(cloudFoundryClient, application, applicationId, stagingTimeout))
+            .then(state -> waitForRunning(cloudFoundryClient, application, applicationId, startupTimeout));
     }
 
     private static Mono<AbstractApplicationResource> stopApplication(CloudFoundryClient cloudFoundryClient, String applicationId) {
@@ -1229,23 +1233,30 @@ private static Mono<Void> uploadApplicationAndWait(CloudFoundryClient cloudFound
 
     }
 
-    private static Mono<String> waitForRunning(CloudFoundryClient cloudFoundryClient, String application, String applicationId) {
+    private static Mono<String> waitForRunning(CloudFoundryClient cloudFoundryClient, String application, String applicationId, Duration startupTimeout) {
+        Duration timeout = Optional.ofNullable(startupTimeout).orElse(Duration.ofMinutes(5));
+
         return requestApplicationInstances(cloudFoundryClient, applicationId)
             .flatMap(response -> Flux.fromIterable(response.values()))
             .map(ApplicationInstanceInfo::getState)
             .reduce("UNKNOWN", collectStates())
             .where(isInstanceComplete())
-            .repeatWhenEmpty(10, DelayUtils.exponentialBackOff(Duration.ofSeconds(1), Duration.ofSeconds(10)))
+            .repeatWhenEmpty(Integer.MAX_VALUE -1, DelayUtils.exponentialBackOff(Duration.ofSeconds(1), Duration.ofSeconds(16), timeout))
             .where(isRunning())
+            .otherwise(throwable -> ExceptionUtils.illegalState("Application %s timed out during start", application))
             .otherwiseIfEmpty(ExceptionUtils.illegalState("Application %s failed during start", application));
     }
 
-    private static Mono<String> waitForStaging(CloudFoundryClient cloudFoundryClient, String application, String applicationId) {
+    private static Mono<String> waitForStaging(CloudFoundryClient cloudFoundryClient, String application, String applicationId, Duration stagingTimeout) {
+        Duration timeout = Optional.ofNullable(stagingTimeout).orElse(Duration.ofMinutes(15));
+
         return requestGetApplication(cloudFoundryClient, applicationId)
             .map(response -> ResourceUtils.getEntity(response).getPackageState())
+            .log("stream.here")
             .where(isStagingComplete())
-            .repeatWhenEmpty(10, DelayUtils.exponentialBackOff(Duration.ofSeconds(1), Duration.ofSeconds(10)))
+            .repeatWhenEmpty(Integer.MAX_VALUE -1, DelayUtils.exponentialBackOff(Duration.ofSeconds(1), Duration.ofSeconds(16), timeout))
             .where(isStaged())
+            .otherwise(throwable -> ExceptionUtils.illegalState("Application %s timed out during staging", application))
             .otherwiseIfEmpty(ExceptionUtils.illegalState("Application %s failed during staging", application));
     }
 

diff --git a/...rc/main/lombok/org/cloudfoundry/operations/applications/CopySourceApplicationRequest.java b/...rc/main/lombok/org/cloudfoundry/operations/applications/CopySourceApplicationRequest.java
@@ -21,6 +21,8 @@
 import org.cloudfoundry.Validatable;
 import org.cloudfoundry.ValidationResult;
 
+import java.time.Duration;
+
 /**
  * The request options for the copy source application operation
  */
@@ -43,6 +45,14 @@ public final class CopySourceApplicationRequest implements Validatable {
      */
     private final Boolean restart;
 
+    /**
+     * How long to wait for startup
+     *
+     * @param startupTimeout how long to wait for startup
+     * @return how long to wait for startup
+     */
+    private final Duration startupTimeout;
+
     /**
      * The name of the target application
      *
@@ -59,6 +69,14 @@ public final class CopySourceApplicationRequest implements Validatable {
      */
     private final String targetOrganization;
 
+    /**
+     * How long to wait for staging
+     *
+     * @param stagingTimeout how long to wait for staging
+     * @return how long to wait for staging
+     */
+    private final Duration stagingTimeout;
+
     /**
      * The space of the target application
      *
@@ -68,15 +86,19 @@ public final class CopySourceApplicationRequest implements Validatable {
     private final String targetSpace;
 
     @Builder
-    CopySourceApplicationRequest(String name, 
+    CopySourceApplicationRequest(String name,
                                  Boolean restart,
+                                 Duration startupTimeout,
                                  String targetName,
                                  String targetOrganization,
+                                 Duration stagingTimeout,
                                  String targetSpace) {
         this.name = name;
         this.restart = restart;
+        this.startupTimeout = startupTimeout;
         this.targetName = targetName;
         this.targetOrganization = targetOrganization;
+        this.stagingTimeout = stagingTimeout;
         this.targetSpace = targetSpace;
     }
 
@@ -91,7 +113,7 @@ public ValidationResult isValid() {
         if (this.targetName == null) {
             builder.message("target application name must be specified");
         }
-        
+
         if (this.targetOrganization != null && this.targetSpace == null) {
             builder.message("target space must be specified with target organization");
         }

diff --git a/...ions/src/main/lombok/org/cloudfoundry/operations/applications/PushApplicationRequest.java b/...ions/src/main/lombok/org/cloudfoundry/operations/applications/PushApplicationRequest.java
@@ -22,6 +22,7 @@
 import org.cloudfoundry.ValidationResult;
 
 import java.io.InputStream;
+import java.time.Duration;
 
 /**
  * The request options for the push application operation
@@ -174,10 +175,26 @@ public final class PushApplicationRequest implements Validatable {
     private final String stack;
 
     /**
-     * The startup timeout in seconds for the application
+     * How long to wait for staging
      *
-     * @param timeout the startup timeout in seconds for the application
-     * @return the startup timeout in seconds for the application
+     * @param stagingTimeout how long to wait for staging
+     * @return how long to wait for staging
+     */
+    private final Duration stagingTimeout;
+
+    /**
+     * How long to wait for startup
+     *
+     * @param startupTimeout how long to wait for startup
+     * @return how long to wait for startup
+     */
+    private final Duration startupTimeout;
+
+    /**
+     * The health check timeout
+     *
+     * @param timeout the health check timeout
+     * @return the health check timeout
      */
     private final Integer timeout;
 
@@ -199,8 +216,10 @@ public final class PushApplicationRequest implements Validatable {
                            String path,
                            Boolean randomRoute,
                            String routePath,
-                           Integer timeout,
-                           String stack) {
+                           String stack,
+                           Duration stagingTimeout,
+                           Duration startupTimeout,
+                           Integer timeout) {
         this.application = application;
         this.buildpack = buildpack;
         this.command = command;
@@ -218,8 +237,10 @@ public final class PushApplicationRequest implements Validatable {
         this.path = path;
         this.randomRoute = randomRoute;
         this.routePath = routePath;
-        this.timeout = timeout;
         this.stack = stack;
+        this.stagingTimeout = stagingTimeout;
+        this.startupTimeout = startupTimeout;
+        this.timeout = timeout;
     }
 
     @Override

diff --git a/...s/src/main/lombok/org/cloudfoundry/operations/applications/RestageApplicationRequest.java b/...s/src/main/lombok/org/cloudfoundry/operations/applications/RestageApplicationRequest.java
@@ -21,17 +21,43 @@
 import org.cloudfoundry.Validatable;
 import org.cloudfoundry.ValidationResult;
 
+import java.time.Duration;
+
 /**
  * The request options for the restage application operation
  */
 @Data
 public final class RestageApplicationRequest implements Validatable {
 
+    /**
+     * The name of the application
+     *
+     * @param name the name of the application
+     * @return the name of the application
+     */
     private final String name;
 
+    /**
+     * How long to wait for staging
+     *
+     * @param stagingTimeout how long to wait for staging
+     * @return how long to wait for staging
+     */
+    private final Duration stagingTimeout;
+
+    /**
+     * How long to wait for startup
+     *
+     * @param startupTimeout how long to wait for startup
+     * @return how long to wait for startup
+     */
+    private final Duration startupTimeout;
+
     @Builder
-    RestageApplicationRequest(String name) {
+    RestageApplicationRequest(String name, Duration stagingTimeout, Duration startupTimeout) {
         this.name = name;
+        this.stagingTimeout = stagingTimeout;
+        this.startupTimeout = startupTimeout;
     }
 
     @Override

diff --git a/...s/src/main/lombok/org/cloudfoundry/operations/applications/RestartApplicationRequest.java b/...s/src/main/lombok/org/cloudfoundry/operations/applications/RestartApplicationRequest.java
@@ -21,17 +21,43 @@
 import org.cloudfoundry.Validatable;
 import org.cloudfoundry.ValidationResult;
 
+import java.time.Duration;
+
 /**
  * The request options for the restart application operation
  */
 @Data
 public final class RestartApplicationRequest implements Validatable {
 
+    /**
+     * The name of the application
+     *
+     * @param name the name of the application
+     * @return the name of the application
+     */
     private final String name;
 
+    /**
+     * How long to wait for staging
+     *
+     * @param stagingTimeout how long to wait for staging
+     * @return how long to wait for staging
+     */
+    private final Duration stagingTimeout;
+
+    /**
+     * How long to wait for startup
+     *
+     * @param startupTimeout how long to wait for startup
+     * @return how long to wait for startup
+     */
+    private final Duration startupTimeout;
+
     @Builder
-    RestartApplicationRequest(String name) {
+    RestartApplicationRequest(String name, Duration stagingTimeout, Duration startupTimeout) {
         this.name = name;
+        this.stagingTimeout = stagingTimeout;
+        this.startupTimeout = startupTimeout;
     }
 
     @Override