feat(provider/kubernetes): allow jobs & pods to "fail" (#2469)

spinnaker · Apr 2, 2018 · cfd5180 · cfd5180
1 parent 0a8d99e
commit cfd5180
Show file tree

Hide file tree

Showing 3 changed files with 52 additions and 2 deletions.
diff --git a/clouddriver-core/src/main/groovy/com/netflix/spinnaker/clouddriver/model/Manifest.java b/clouddriver-core/src/main/groovy/com/netflix/spinnaker/clouddriver/model/Manifest.java
@@ -34,6 +34,14 @@ class Status {
     Condition stable = Condition.builder().state(true).build();
     Condition paused = Condition.builder().state(false).build();
     Condition available = Condition.builder().state(true).build();
+    Condition failed = Condition.builder().state(false).build();
+
+    public Status failed(String message) {
+      failed.setMessage(message);
+      failed.setState(true);
+
+      return this;
+    }
 
     public Status unstable(String message) {
       stable.setMessage(message);

diff --git a/...oovy/com/netflix/spinnaker/clouddriver/kubernetes/v2/op/handler/KubernetesJobHandler.java b/...oovy/com/netflix/spinnaker/clouddriver/kubernetes/v2/op/handler/KubernetesJobHandler.java
@@ -26,10 +26,13 @@
 import com.netflix.spinnaker.clouddriver.kubernetes.v2.description.manifest.KubernetesManifest;
 import com.netflix.spinnaker.clouddriver.model.Manifest.Status;
 import io.kubernetes.client.models.V1Job;
+import io.kubernetes.client.models.V1JobCondition;
 import io.kubernetes.client.models.V1JobSpec;
 import io.kubernetes.client.models.V1JobStatus;
 import org.springframework.stereotype.Component;
 
+import java.util.List;
+
 @Component
 public class KubernetesJobHandler extends KubernetesHandler implements
   CanDelete,
@@ -88,9 +91,18 @@ private Status status(V1Job job) {
       succeeded = status.getSucceeded();
     }
     if (succeeded < completions) {
-      return result.unstable("Waiting for jobs to finish");
+      List<V1JobCondition> conditions = status.getConditions();
+      if (conditions != null && conditions.stream().anyMatch(this::jobDeadlineExceeded)) {
+        return result.failed("Job deadline exceeded");
+      } else {
+        return result.unstable("Waiting for jobs to finish");
+      }
     }
 
     return result;
   }
+
+  private boolean jobDeadlineExceeded(V1JobCondition condition) {
+    return "DeadlineExceeded".equalsIgnoreCase(condition.getReason()) && "True".equalsIgnoreCase(condition.getStatus());
+  }
 }
diff --git a/...oovy/com/netflix/spinnaker/clouddriver/kubernetes/v2/op/handler/KubernetesPodHandler.java b/...oovy/com/netflix/spinnaker/clouddriver/kubernetes/v2/op/handler/KubernetesPodHandler.java
@@ -20,13 +20,16 @@
 import com.netflix.spinnaker.clouddriver.kubernetes.v2.artifact.ArtifactReplacer;
 import com.netflix.spinnaker.clouddriver.kubernetes.v2.artifact.ArtifactTypes;
 import com.netflix.spinnaker.clouddriver.kubernetes.v2.caching.Keys;
+import com.netflix.spinnaker.clouddriver.kubernetes.v2.caching.agent.KubernetesCacheDataConverter;
 import com.netflix.spinnaker.clouddriver.kubernetes.v2.caching.agent.KubernetesPodCachingAgent;
 import com.netflix.spinnaker.clouddriver.kubernetes.v2.caching.agent.KubernetesV2CachingAgent;
 import com.netflix.spinnaker.clouddriver.kubernetes.v2.caching.view.provider.KubernetesCacheUtils;
 import com.netflix.spinnaker.clouddriver.kubernetes.v2.description.KubernetesSpinnakerKindMap.SpinnakerKind;
 import com.netflix.spinnaker.clouddriver.kubernetes.v2.description.manifest.KubernetesKind;
 import com.netflix.spinnaker.clouddriver.kubernetes.v2.description.manifest.KubernetesManifest;
 import com.netflix.spinnaker.clouddriver.model.Manifest.Status;
+import io.kubernetes.client.models.V1Pod;
+import io.kubernetes.client.models.V1PodStatus;
 import org.springframework.stereotype.Component;
 
 import java.util.Map;
@@ -60,7 +63,34 @@ public SpinnakerKind spinnakerKind() {
 
   @Override
   public Status status(KubernetesManifest manifest) {
-    return new Status();
+    Status result = new Status();
+    V1Pod pod = KubernetesCacheDataConverter.getResource(manifest, V1Pod.class);
+    V1PodStatus status = pod.getStatus();
+
+    if (status == null) {
+      result.unstable("No status reported yet")
+          .unavailable("No availability reported");
+      return result;
+    }
+
+    // https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/
+    String phase = status.getPhase();
+
+    if (phase == null ) {
+      result.unstable("No phase reported yet")
+          .unavailable("No availability reported");
+    } else if (phase.equals("pending")) {
+      result.unstable("Pod is 'pending'")
+          .unavailable("Pod has not been scheduled yet");
+    } else if (phase.equals("unknown")) {
+      result.unstable("Pod has 'unknown' phase")
+          .unavailable("No availability reported");
+    } else if (phase.equals("failed")) {
+      result.failed("Pod has 'failed'")
+          .unavailable("Pod is not running");
+    }
+
+    return result;
   }
 
   @Override