rh-che eclipse-che#557: Adding handler for stopping k8s / openshift r…

…untime if unrecoverable event occurs during workspace startup

Signed-off-by: Ilya Buziuk <ibuziuk@redhat.com>

infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/KubernetesInternalRuntime.java

@@ -231,6 +231,8 @@ private void waitMachines(
       failure.completeExceptionally(ex);
       cancelAll(toCancelFutures);
       wrapAndRethrow(ex.getCause());
+    } finally {
+      namespace.pods().stopWatch();
     }
   }
 
@@ -508,6 +510,26 @@ public void accept(ProbeResult probeResult) {
     }
   }
 
+  /** Listens container's events and terminates runtime if unrecoverable event occurs. */
+  public class UnrecoverableEventHanler implements ContainerEventHandler {
+
+    @Override
+    public void handle(ContainerEvent event) {
+      String reason = event.getReason();
+      String message = event.getMessage();
+      if (isUnrecoverable(reason)) {
+        LOG.error("Unrecoverable event occured {}, {}", reason, message);
+        eventPublisher.sendRuntimeStoppedEvent(
+            format("Unrecoverable event occured: '%s', '%s'", reason, message),
+            getContext().getIdentity());
+      }
+    }
+
+    private boolean isUnrecoverable(String reason) {
+      return reason.equals("Failed Mount") || reason.equals("Failed Scheduling");
+    }
+  }
+
   /** Listens container's events and publish them as machine logs. */
   class MachineLogsPublisher implements ContainerEventHandler {
 

infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/namespace/KubernetesPods.java

@@ -325,6 +325,7 @@ public void eventReceived(Action action, Event event) {
                       new ContainerEvent(
                           podName,
                           containerName,
+                          event.getReason(),
                           event.getMessage(),
                           event.getMetadata().getCreationTimestamp());
                   containerEventsHandlers.forEach(h -> h.handle(containerEvent));
@@ -346,7 +347,7 @@ public void onClose(KubernetesClientException ignored) {}
   }
 
   /** Stops watching the pods inside Kubernetes namespace. */
-  void stopWatch() {
+  public void stopWatch() {
     try {
       if (podWatch != null) {
         podWatch.close();

infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/namespace/event/ContainerEvent.java

@@ -21,12 +21,15 @@
 public class ContainerEvent {
   private final String podName;
   private final String containerName;
+  private final String reason;
   private final String message;
   private final String time;
 
-  public ContainerEvent(String podName, String containerName, String message, String time) {
+  public ContainerEvent(
+      String podName, String containerName, String reason, String message, String time) {
     this.podName = podName;
     this.containerName = containerName;
+    this.reason = reason;
     this.message = message;
     this.time = time;
   }
@@ -41,6 +44,11 @@ public String getContainerName() {
     return containerName;
   }
 
+  /** Returns the reason of the event. */
+  public String getReason() {
+    return reason;
+  }
+
   /** Returns the contents of the event. */
   public String getMessage() {
     return message;
@@ -80,6 +88,9 @@ public String toString() {
         + ", containerName='"
         + containerName
         + '\''
+        + ", reason='"
+        + reason
+        + '\''
         + ", message='"
         + message
         + '\''

infrastructures/openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/OpenShiftInternalRuntime.java

@@ -88,6 +88,8 @@ protected void startMachines() throws InfrastructureException {
     // project.pods().watch(new AbnormalStopHandler());
     // project.pods().watchContainers(new MachineLogsPublisher());
 
+    project.pods().watchContainers(new UnrecoverableEventHanler());
+
     doStartMachine(new OpenShiftServerResolver(createdServices, createdRoutes));
   }
 }