Introduce wsagent ping success threshold

Signed-off-by: Mario Loriedo <mloriedo@redhat.com>
eclipse-che · Dec 7, 2017 · b70ddb6 · b70ddb6
1 parent 7416587
commit b70ddb6
Show file tree

Hide file tree

Showing 3 changed files with 42 additions and 11 deletions.
diff --git a/assembly/assembly-wsmaster-war/src/main/webapp/WEB-INF/classes/codenvy/che.properties b/assembly/assembly-wsmaster-war/src/main/webapp/WEB-INF/classes/codenvy/che.properties
@@ -110,6 +110,7 @@ che.workspace.che_server_endpoint=http://che-host:${SERVER_PORT}/wsmaster/api
 che.workspace.agent.dev.max_start_time_ms=180000
 che.workspace.agent.dev.ping_delay_ms=2000
 che.workspace.agent.dev.ping_conn_timeout_ms=2000
+che.workspace.agent.dev.ping_success_threshold=1
 che.workspace.agent.dev.ping_timeout_error_msg=Timeout. The Che server is unable to ping your workspace. This implies a network configuration issue, workspace boot failure, or an unusually slow workspace boot.
 
 che.agent.dev.max_start_time_ms=120000

diff --git a/wsagent/agent/src/main/java/org/eclipse/che/api/agent/WsAgentLauncher.java b/wsagent/agent/src/main/java/org/eclipse/che/api/agent/WsAgentLauncher.java
@@ -54,6 +54,7 @@ public class WsAgentLauncher implements AgentLauncher {
   private final long wsAgentMaxStartTimeMs;
   private final long wsAgentPingDelayMs;
   private final String pingTimedOutErrorMessage;
+  private final int wsAgentPingSuccessThreshold;
   private final String wsAgentRunCommand;
 
   @Inject
@@ -64,12 +65,14 @@ public WsAgentLauncher(
       @Nullable @Named("machine.ws_agent.run_command") String wsAgentRunCommand,
       @Named("che.workspace.agent.dev.max_start_time_ms") long wsAgentMaxStartTimeMs,
       @Named("che.workspace.agent.dev.ping_delay_ms") long wsAgentPingDelayMs,
+      @Named("che.workspace.agent.dev.ping_success_treshold") int wsAgentPingSuccessThreshold,
       @Named("che.workspace.agent.dev.ping_timeout_error_msg") String pingTimedOutErrorMessage) {
     this.machineProcessManagerProvider = machineProcessManagerProvider;
     this.wsAgentPingRequestFactory = wsAgentPingRequestFactory;
     this.wsAgentHealthChecker = wsAgentHealthChecker;
     this.wsAgentMaxStartTimeMs = wsAgentMaxStartTimeMs;
     this.wsAgentPingDelayMs = wsAgentPingDelayMs;
+    this.wsAgentPingSuccessThreshold = wsAgentPingSuccessThreshold;
     this.pingTimedOutErrorMessage = pingTimedOutErrorMessage;
     this.wsAgentRunCommand = wsAgentRunCommand;
   }
@@ -121,12 +124,19 @@ public void launch(Instance machine, Agent agent) throws ServerException {
           wsAgentPingUrl,
           pingStartTimestamp);
 
+      int pingSuccess = 0;
       while (System.currentTimeMillis() - pingStartTimestamp < wsAgentMaxStartTimeMs) {
         if (pingWsAgent(machine)) {
-          return;
+          pingSuccess++;
         } else {
-          Thread.sleep(wsAgentPingDelayMs);
+          pingSuccess = 0;
         }
+
+        if (pingSuccess == wsAgentPingSuccessThreshold) {
+          return;
+        }
+
+        Thread.sleep(wsAgentPingDelayMs);
       }
     } catch (BadRequestException | ServerException | NotFoundException e) {
       throw new ServerException(e.getServiceError());

diff --git a/wsagent/agent/src/test/java/org/eclipse/che/api/agent/WsAgentLauncherTest.java b/wsagent/agent/src/test/java/org/eclipse/che/api/agent/WsAgentLauncherTest.java
@@ -52,6 +52,7 @@ public class WsAgentLauncherTest {
   private static final String WS_AGENT_PORT = Constants.WS_AGENT_PORT;
   private static final long WS_AGENT_MAX_START_TIME_MS = 1000;
   private static final long WS_AGENT_PING_DELAY_MS = 1;
+  private static final int WS_AGENT_PING_SUCCESS_THRESHOLD = 5;
   private static final String WS_AGENT_SERVER_LOCATION = "ws-agent.com:456789/";
   private static final String WS_AGENT_SERVER_URL = "http://" + WS_AGENT_SERVER_LOCATION;
   private static final String WS_AGENT_SERVER_LOCATION_EXT = "ws-agent-ext.com:456789/";
@@ -84,6 +85,7 @@ public void setUp() throws Exception {
             null,
             WS_AGENT_MAX_START_TIME_MS,
             WS_AGENT_PING_DELAY_MS,
+            WS_AGENT_PING_SUCCESS_THRESHOLD,
             WS_AGENT_TIMED_OUT_MESSAGE);
     pingRequest = Mockito.mock(HttpJsonRequest.class, new SelfReturningAnswer());
     Mockito.when(agent.getScript()).thenReturn("script");
@@ -119,8 +121,9 @@ public void shouldStartWsAgentUsingMachineExec() throws Exception {
   public void shouldPingWsAgentAfterStart() throws Exception {
     wsAgentLauncher.launch(machine, agent);
 
-    Mockito.verify(wsAgentHealthChecker).check(any(Machine.class));
-    Mockito.verify(pingResponse).getCode();
+    Mockito.verify(wsAgentHealthChecker, Mockito.times(WS_AGENT_PING_SUCCESS_THRESHOLD))
+        .check(any(Machine.class));
+    Mockito.verify(pingResponse, Mockito.times(WS_AGENT_PING_SUCCESS_THRESHOLD)).getCode();
   }
 
   @Test
@@ -131,8 +134,9 @@ public void shouldPingWsAgentMultipleTimesAfterStartIfPingFailsWithException() t
 
     wsAgentLauncher.launch(machine, agent);
 
-    Mockito.verify(wsAgentHealthChecker, Mockito.times(2)).check(any(Machine.class));
-    Mockito.verify(pingResponse).getCode();
+    Mockito.verify(wsAgentHealthChecker, Mockito.times(1 + WS_AGENT_PING_SUCCESS_THRESHOLD))
+        .check(any(Machine.class));
+    Mockito.verify(pingResponse, Mockito.times(WS_AGENT_PING_SUCCESS_THRESHOLD)).getCode();
   }
 
   @Test
@@ -146,20 +150,22 @@ public void shouldPingWsAgentMultipleTimesAfterStartIfPingReturnsNotOKResponseCo
 
     wsAgentLauncher.launch(machine, agent);
 
-    Mockito.verify(wsAgentHealthChecker, Mockito.times(3)).check(any(Machine.class));
-    Mockito.verify(pingResponse, Mockito.times(3)).getCode();
+    Mockito.verify(wsAgentHealthChecker, Mockito.times(2 + WS_AGENT_PING_SUCCESS_THRESHOLD))
+        .check(any(Machine.class));
+    Mockito.verify(pingResponse, Mockito.times(2 + WS_AGENT_PING_SUCCESS_THRESHOLD)).getCode();
   }
 
   @Test
-  public void shouldNotPingWsAgentAfterFirstSuccessfulPing() throws Exception {
+  public void shouldNotPingWsAgentAfterSuccessThresholdPing() throws Exception {
     Mockito.when(wsAgentHealthChecker.check(any(Machine.class)))
         .thenThrow(new ServerException(""))
         .thenReturn(pingResponse);
 
     wsAgentLauncher.launch(machine, agent);
 
-    Mockito.verify(wsAgentHealthChecker, Mockito.times(2)).check(any(Machine.class));
-    Mockito.verify(pingResponse).getCode();
+    Mockito.verify(wsAgentHealthChecker, Mockito.times(1 + WS_AGENT_PING_SUCCESS_THRESHOLD))
+        .check(any(Machine.class));
+    Mockito.verify(pingResponse, Mockito.times(WS_AGENT_PING_SUCCESS_THRESHOLD)).getCode();
   }
 
   @Test(
@@ -219,4 +225,18 @@ public void shouldThrowMachineExceptionIfPingsWereUnsuccessfulTooLong() throws E
 
     wsAgentLauncher.launch(machine, agent);
   }
+
+  @Test
+  public void shouldContinuePingWsAgentAfterNotConsecutiveSuccessThresholdPing() throws Exception {
+    Mockito.when(wsAgentHealthChecker.check(any(Machine.class)))
+        .thenReturn(pingResponse)
+        .thenThrow(new ServerException(""))
+        .thenReturn(pingResponse);
+
+    wsAgentLauncher.launch(machine, agent);
+
+    Mockito.verify(wsAgentHealthChecker, Mockito.times(2 + WS_AGENT_PING_SUCCESS_THRESHOLD))
+        .check(any(Machine.class));
+    Mockito.verify(pingResponse, Mockito.times(1 + WS_AGENT_PING_SUCCESS_THRESHOLD)).getCode();
+  }
 }