Fixes eclipse-che#14781 - Add additional diagnostic for failure when …

…K8s pronounces a pod running. It still might have failed if one of the containers has terminated. Signed-off-by: Lukas Krejci <lkrejci@redhat.com>
metlos · Dec 10, 2019 · 929c7a1 · 929c7a1
1 parent b2886fa
commit 929c7a1
Show file tree

Hide file tree

Showing 2 changed files with 67 additions and 6 deletions.
diff --git a/.../org/eclipse/che/workspace/infrastructure/kubernetes/namespace/KubernetesDeployments.java b/.../org/eclipse/che/workspace/infrastructure/kubernetes/namespace/KubernetesDeployments.java
@@ -22,13 +22,16 @@
 import static org.eclipse.che.workspace.infrastructure.kubernetes.namespace.KubernetesObjectUtil.setSelector;
 
 import com.google.common.base.Strings;
+import io.fabric8.kubernetes.api.model.ContainerStateTerminated;
+import io.fabric8.kubernetes.api.model.ContainerStatus;
 import io.fabric8.kubernetes.api.model.DoneablePod;
 import io.fabric8.kubernetes.api.model.Event;
 import io.fabric8.kubernetes.api.model.ObjectMeta;
 import io.fabric8.kubernetes.api.model.ObjectReference;
 import io.fabric8.kubernetes.api.model.OwnerReference;
 import io.fabric8.kubernetes.api.model.Pod;
 import io.fabric8.kubernetes.api.model.PodSpec;
+import io.fabric8.kubernetes.api.model.PodStatus;
 import io.fabric8.kubernetes.api.model.apps.Deployment;
 import io.fabric8.kubernetes.api.model.apps.DeploymentBuilder;
 import io.fabric8.kubernetes.api.model.apps.DoneableDeployment;
@@ -46,6 +49,7 @@
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Date;
+import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
@@ -388,19 +392,46 @@ public void onClose(KubernetesClientException cause) {
   }
 
   private void handleStartingPodStatus(CompletableFuture<Void> podRunningFuture, Pod pod) {
-    if (POD_STATUS_PHASE_RUNNING.equals(pod.getStatus().getPhase())) {
-      podRunningFuture.complete(null);
+    PodStatus status = pod.getStatus();
+    String podPhase = status.getPhase();
+    if (POD_STATUS_PHASE_RUNNING.equals(podPhase)) {
+      // check that all the containers are ready...
+      Map<String, String> terminatedContainers = new HashMap<>();
+
+      for (ContainerStatus cs : status.getContainerStatuses()) {
+        ContainerStateTerminated terminated = cs.getState().getTerminated();
+        if (terminated != null) {
+          terminatedContainers.put(
+              cs.getName(),
+              "reason = " + terminated.getReason() + ", message = " + terminated.getMessage());
+        }
+      }
+
+      if (terminatedContainers.isEmpty()) {
+        podRunningFuture.complete(null);
+      } else {
+        String errorMessage =
+            "The following containers have terminated:\n"
+                + terminatedContainers
+                    .entrySet()
+                    .stream()
+                    .map(e -> e.getKey() + ": " + e.getValue())
+                    .collect(Collectors.joining("" + "\n"));
+
+        podRunningFuture.completeExceptionally(new InfrastructureException(errorMessage));
+      }
+
       return;
     }
 
-    if (POD_STATUS_PHASE_SUCCEEDED.equals(pod.getStatus().getPhase())) {
+    if (POD_STATUS_PHASE_SUCCEEDED.equals(podPhase)) {
       podRunningFuture.completeExceptionally(
           new InfrastructureException(
               "Pod container has been terminated. Container must be configured to use a non-terminating command."));
       return;
     }
 
-    if (POD_STATUS_PHASE_FAILED.equals(pod.getStatus().getPhase())) {
+    if (POD_STATUS_PHASE_FAILED.equals(podPhase)) {
       String exceptionMessage = "Pod '" + pod.getMetadata().getName() + "' failed to start.";
       String reason = pod.getStatus().getReason();
       if (Strings.isNullOrEmpty(reason)) {

diff --git a/.../eclipse/che/workspace/infrastructure/kubernetes/namespace/KubernetesDeploymentsTest.java b/.../eclipse/che/workspace/infrastructure/kubernetes/namespace/KubernetesDeploymentsTest.java
@@ -30,6 +30,8 @@
 import static org.testng.Assert.fail;
 
 import com.google.common.collect.ImmutableMap;
+import io.fabric8.kubernetes.api.model.ContainerStateBuilder;
+import io.fabric8.kubernetes.api.model.ContainerStatus;
 import io.fabric8.kubernetes.api.model.DoneableEvent;
 import io.fabric8.kubernetes.api.model.DoneablePod;
 import io.fabric8.kubernetes.api.model.Event;
@@ -236,10 +238,10 @@ public void shouldThrowExceptionWhenMultiplePodsExistsForDeploymentsOnPodFetchin
   }
 
   @Test
-  public void shouldCompleteFutureForWaitingPidIfStatusIsRunning() {
+  public void shouldCompleteFutureForWaitingPodIfStatusIsRunning() {
     // given
     when(status.getPhase()).thenReturn(POD_STATUS_PHASE_RUNNING);
-    CompletableFuture future = kubernetesDeployments.waitRunningAsync(POD_NAME);
+    CompletableFuture<?> future = kubernetesDeployments.waitRunningAsync(POD_NAME);
 
     // when
     verify(podResource).watch(watcherCaptor.capture());
@@ -250,6 +252,34 @@ public void shouldCompleteFutureForWaitingPidIfStatusIsRunning() {
     assertTrue(future.isDone());
   }
 
+  @Test
+  public void
+      shouldCompleteExceptionallyFutureForWaitingPodIfStatusIsRunningButSomeContainersAreTerminated() {
+    // given
+    ContainerStatus containerStatus = mock(ContainerStatus.class);
+    when(containerStatus.getName()).thenReturn("FailingContainer");
+    when(containerStatus.getState())
+        .thenReturn(
+            new ContainerStateBuilder()
+                .withNewTerminated()
+                .withReason("Completed")
+                .endTerminated()
+                .build());
+
+    when(status.getPhase()).thenReturn(POD_STATUS_PHASE_RUNNING);
+    when(status.getContainerStatuses()).thenReturn(singletonList(containerStatus));
+    CompletableFuture<?> future = kubernetesDeployments.waitRunningAsync(POD_NAME);
+
+    // when
+    verify(podResource).watch(watcherCaptor.capture());
+    Watcher<Pod> watcher = watcherCaptor.getValue();
+    watcher.eventReceived(Watcher.Action.MODIFIED, pod);
+
+    // then
+    assertTrue(future.isDone());
+    assertTrue(future.isCompletedExceptionally());
+  }
+
   @Test
   public void shouldCompleteExceptionallyFutureForWaitingPodIfStatusIsSucceeded() throws Exception {
     // given