Merge pull request #317 from Shopify/improve-preemption-handling

Don't consider pod preempting a failure
Shopify · Jul 17, 2018 · dc5fd47 · dc5fd47
2 parents 435b545 + ab83055
commit dc5fd47
Show file tree

Hide file tree

Showing 2 changed files with 20 additions and 1 deletion.
diff --git a/lib/kubernetes-deploy/kubernetes_resource/pod.rb b/lib/kubernetes-deploy/kubernetes_resource/pod.rb
@@ -4,6 +4,10 @@ class Pod < KubernetesResource
     TIMEOUT = 10.minutes
 
     FAILED_PHASE_NAME = "Failed"
+    TRANSIENT_FAILURE_REASONS = %w(
+      Evicted
+      Preempting
+    )
 
     def initialize(namespace:, context:, definition:, logger:,
       statsd_tags: nil, parent: nil, deploy_started_at: nil)
@@ -57,7 +61,7 @@ def timeout_message
     end
 
     def failure_message
-      if phase == FAILED_PHASE_NAME && reason != "Evicted"
+      if phase == FAILED_PHASE_NAME && !TRANSIENT_FAILURE_REASONS.include?(reason)
         phase_problem = "Pod status: #{status}. "
       end
 

diff --git a/test/unit/kubernetes-deploy/kubernetes_resource/pod_test.rb b/test/unit/kubernetes-deploy/kubernetes_resource/pod_test.rb
@@ -170,6 +170,21 @@ def test_deploy_failed_is_false_for_evicted
     assert_nil pod.failure_message
   end
 
+  def test_deploy_failed_is_false_for_preempting
+    container_state = pod_spec.merge(
+      "status" => {
+        "message" => "Preempted in order to admit critical pod",
+        "phase" => "Failed",
+        "reason" => "Preempting",
+        "startTime" => "2018-04-13T22:43:23Z"
+      }
+    )
+    pod = build_synced_pod(container_state)
+
+    refute_predicate pod, :deploy_failed?
+    assert_nil pod.failure_message
+  end
+
   private
 
   def pod_spec