Support timeout override for deployments

README.md

@@ -226,7 +226,7 @@ This is a limitation of the current implementation.
 ### Customizing behaviour with annotations
 - `kubernetes-deploy.shopify.io/timeout-override`: Override the tool's hard timeout for one specific resource. Both full ISO8601 durations and the time portion of ISO8601 durations are valid. Value must be between 1 second and 24 hours.
   - _Example values_: 45s / 3m / 1h / PT0.25H
-  - _Compatibility_: all resource types (Note: `Deployment` timeouts are based on `spec.progressDeadlineSeconds` if present, and that field has a default value as of the `apps/v1beta1` group version. Using this annotation will have no effect on `Deployment`s that time out with "Timeout reason: ProgressDeadlineExceeded".)
+  - _Compatibility_: all resource types
 - `kubernetes-deploy.shopify.io/required-rollout`: Modifies how much of the rollout needs to finish
 before the deployment is considered successful.
   - _Compatibility_: Deployment

lib/kubernetes-deploy/kubernetes_resource/deployment.rb

@@ -63,7 +63,9 @@ def failure_message
     end
 
     def timeout_message
-      reason_msg = if progress_condition.present?
+      reason_msg = if timeout_override
+        STANDARD_TIMEOUT_MESSAGE
+      elsif progress_condition.present?
         "Timeout reason: #{progress_condition['reason']}"
       else
         "Timeout reason: hard deadline for #{type}"
@@ -73,11 +75,19 @@ def timeout_message
     end
 
     def pretty_timeout_type
-      progress_deadline.present? ? "progress deadline: #{progress_deadline}s" : super
+      if timeout_override
+        "timeout override: #{timeout_override}s"
+      elsif progress_deadline.present?
+        "progress deadline: #{progress_deadline}s"
+      else
+        super
+      end
     end
 
     def deploy_timed_out?
       return false if deploy_failed?
+      return super if timeout_override
+
       # Do not use the hard timeout if progress deadline is set
       progress_condition.present? ? deploy_failing_to_progress? : super
     end

test/fixtures/invalid/bad_probe.yml

@@ -2,8 +2,6 @@ apiVersion: extensions/v1beta1 # keep as long as this group is valid
 kind: Deployment
 metadata:
   name: bad-probe
-  annotations:
-    kubernetes-deploy.shopify.io/timeout-override: '25s'
 spec:
   replicas: 1
   # progressDeadlineSeconds: 10 -- do not add: this is used to test the hard deadline

test/integration/kubernetes_deploy_test.rb

@@ -446,6 +446,21 @@ def test_deployment_with_progress_times_out_for_short_duration
     ])
   end
 
+  def test_deployment_with_timeout_override
+    result = deploy_fixtures("long-running", subset: ['undying-deployment.yml.erb']) do |fixtures|
+      deployment = fixtures['undying-deployment.yml.erb']['Deployment'].first
+      deployment['spec']['progressDeadlineSeconds'] = 5
+      deployment["metadata"]["annotations"] = {
+        KubernetesDeploy::KubernetesResource::TIMEOUT_OVERRIDE_ANNOTATION => "10S",
+      }
+      container = deployment['spec']['template']['spec']['containers'].first
+      container['readinessProbe'] = { "exec" => { "command" => ['- ls'] } }
+    end
+    assert_deploy_failure(result, :timed_out)
+    assert_logs_match_all(KubernetesDeploy::KubernetesResource::STANDARD_TIMEOUT_MESSAGE.split("\n") +
+      ["timeout override: 10s"])
+  end
+
   def test_wait_false_ignores_non_priority_resource_failures
     # web depends on configmap so will not succeed deployed alone
     assert_deploy_success(deploy_fixtures("hello-cloud", subset: ["web.yml.erb"], wait: false))

test/unit/kubernetes-deploy/kubernetes_resource/deployment_test.rb

@@ -273,6 +273,7 @@ def test_deploy_timed_out_with_hard_timeout
         template: build_deployment_template(status: { "replicas" => 3, "conditions" => [] }),
         replica_sets: [build_rs_template(status: { "replica" => 1 })]
       )
+
       deploy.deploy_started_at = Time.now.utc - KubernetesDeploy::Deployment::TIMEOUT
       refute deploy.deploy_timed_out?
 
@@ -307,6 +308,39 @@ def test_deploy_timed_out_based_on_progress_deadline
     end
   end
 
+  def test_deploy_timed_out_based_on_timeout_override
+    Timecop.freeze do
+      template = build_deployment_template(
+        status: {
+          "replicas" => 3,
+          "observedGeneration" => 2,
+          "conditions" => [{
+            "type" => "Progressing",
+            "status" => 'False',
+            "lastUpdateTime" => Time.now.utc - 10.seconds,
+            "reason" => "ProgressDeadlineExceeded",
+          }],
+        }
+      )
+      template["metadata"]["annotations"][KubernetesDeploy::KubernetesResource::TIMEOUT_OVERRIDE_ANNOTATION] = "15S"
+      template["spec"]["progressDeadlineSeconds"] = "10"
+      deploy = build_synced_deployment(
+        template: template,
+        replica_sets: [build_rs_template(status: { "replica" => 1 })]
+      )
+
+      assert_equal(deploy.timeout, 15)
+      refute(deploy.deploy_timed_out?, "Deploy not started shouldn't have timed out")
+      deploy.deploy_started_at = Time.now.utc - 11.seconds
+      refute(deploy.deploy_timed_out?, "Deploy should not timeout based on progressDeadlineSeconds")
+      deploy.deploy_started_at = Time.now.utc - 16.seconds
+      assert(deploy.deploy_timed_out?, "Deploy should timeout according to timoeout override")
+      assert_equal(KubernetesDeploy::KubernetesResource::STANDARD_TIMEOUT_MESSAGE + "\nLatest ReplicaSet: web-1",
+        deploy.timeout_message.strip)
+      assert_equal(deploy.pretty_timeout_type, "timeout override: 15s")
+    end
+  end
+
   def test_deploy_timed_out_based_on_progress_deadline_ignores_statuses_for_older_generations
     Timecop.freeze do
       deployment_status = {