Merge pull request #883 from Shopify/evicted-pod

Ignore Evicted Pods in DaemonSet deployments
Shopify · Apr 1, 2022 · 8c7fca3 · 8c7fca3
2 parents 228fdae + 50ab7c0
commit 8c7fca3
Show file tree

Hide file tree

Showing 3 changed files with 31 additions and 1 deletion.
diff --git a/lib/krane/kubernetes_resource/daemon_set.rb b/lib/krane/kubernetes_resource/daemon_set.rb
@@ -59,7 +59,7 @@ def initialize(definition:)
     def relevant_pods_ready?
       return true if rollout_data["desiredNumberScheduled"].to_i == rollout_data["numberReady"].to_i # all pods ready
       relevant_node_names = @nodes.map(&:name)
-      considered_pods = @pods.select { |p| relevant_node_names.include?(p.node_name) }
+      considered_pods = @pods.select { |p| relevant_node_names.include?(p.node_name) && !p.evicted? }
       @logger.debug("DaemonSet is reporting #{rollout_data['numberReady']} pods ready." \
         " Considered #{considered_pods.size} pods out of #{@pods.size} for #{@nodes.size} nodes.")
       considered_pods.present? &&

diff --git a/lib/krane/kubernetes_resource/pod.rb b/lib/krane/kubernetes_resource/pod.rb
@@ -105,6 +105,10 @@ def node_name
       @instance_data.dig('spec', 'nodeName')
     end
 
+    def evicted?
+      phase == "Failed" && reason == "Evicted"
+    end
+
     private
 
     def failed_schedule_reason

diff --git a/test/unit/krane/kubernetes_resource/daemon_set_test.rb b/test/unit/krane/kubernetes_resource/daemon_set_test.rb
@@ -126,6 +126,32 @@ def test_deploy_passes_when_nodes_not_ready
     assert_predicate(ds, :deploy_succeeded?)
   end
 
+  def test_deploy_passes_when_pod_evicted
+    status = {
+      "desiredNumberScheduled": 3,
+      "updatedNumberScheduled": 3,
+      "numberReady": 2,
+    }
+    ds_template = build_ds_template(filename: 'daemon_set.yml', status: status)
+    pod_templates = load_fixtures(filenames: ['daemon_set_pods.yml'])
+    node_templates = load_fixtures(filenames: ['nodes.yml'])
+    ds = build_synced_ds(ds_template: ds_template, pod_templates: pod_templates, node_templates: node_templates)
+    refute_predicate(ds, :deploy_succeeded?)
+
+    pod_templates[2]["status"] = {
+      "message": "Pod The node had condition: [DiskPressure].",
+      "phase": "Failed",
+      "reason": "Evicted",
+      "startTime": "2022-03-31T20:14:06Z"
+    }
+
+    stub_kind_get("DaemonSet", items: [ds_template])
+    stub_kind_get("Pod", items: pod_templates)
+    stub_kind_get("Node", items: node_templates, use_namespace: false)
+    ds.sync(build_resource_cache)
+    assert_predicate(ds, :deploy_succeeded?)
+  end
+
   def test_deploy_fails_when_not_all_pods_updated
     status = {
       "desiredNumberScheduled": 2,