From 6b7bbd8904bfed89f433de4c6b40a60d9df94915 Mon Sep 17 00:00:00 2001 From: Daniel Turner Date: Thu, 14 Dec 2017 11:42:08 -0500 Subject: [PATCH] add a restart test --- .../kubernetes_resource/deployment.rb | 55 ++++++++++--------- .../kubernetes_resource/replica_set.rb | 2 +- test/fixtures/slow-cloud/web.yml.erb | 7 ++- test/integration/kubernetes_deploy_test.rb | 26 +++------ test/integration/restart_task_test.rb | 21 +++++++ 5 files changed, 66 insertions(+), 45 deletions(-) diff --git a/lib/kubernetes-deploy/kubernetes_resource/deployment.rb b/lib/kubernetes-deploy/kubernetes_resource/deployment.rb index 97a394ff4..b9dfffa69 100644 --- a/lib/kubernetes-deploy/kubernetes_resource/deployment.rb +++ b/lib/kubernetes-deploy/kubernetes_resource/deployment.rb @@ -111,31 +111,6 @@ def running_rs end end - def min_unavailable_rollout - @definition.dig('metadata', 'annotations', 'kubernetes-deploy.shopify.io/min-unavailable-rollout') - end - - def minimum_unavailable_replicas_to_succeeded - desired = @desired_replicas - - if min_unavailable_rollout =~ /%/ - desired *= (100 - min_unavailable_rollout.to_i) / 100.0 - else - desired = min_unavailable_rollout.to_i - end - - desired.to_i - end - - def partial_deployment_success - minimum_needed = minimum_unavailable_replicas_to_succeeded - - running_rs.size <= 2 && - @latest_rs.desired_replicas > minimum_needed && - @rollout_data["updatedReplicas"].to_i >= minimum_needed && - @rollout_data["availableReplicas"].to_i >= minimum_needed - end - def find_latest_rs current_revision = @deployment_data["metadata"]["annotations"]["deployment.kubernetes.io/revision"] @@ -157,5 +132,35 @@ def find_latest_rs rs.sync(latest_rs_data) rs end + + def min_unavailable_rollout + @definition.dig('metadata', 'annotations', 'kubernetes-deploy.shopify.io/required-rollout') + end + + def minimum_unavailable_replicas_to_succeeded + desired = @desired_replicas + + case min_unavailable_rollout + when 'maxUnavailable' + max_unavailable = @definition.dig('spec', 'strategy', 'rollingUpdate', 'maxUnavailable') + if max_unavailable =~ /%/ + (desired * (100 - max_unavailable.to_i) / 100.0).to_i + else + desired - max_unavailable.to_i + end + when 'none' + 0 + else + raise "#{min_unavailable_rollout} is not a valid value for required-rollout" + end + end + + def partial_deployment_success + minimum_needed = minimum_unavailable_replicas_to_succeeded + + @latest_rs.desired_replicas >= minimum_needed && + @latest_rs.rollout_data["readyReplicas"].to_i >= minimum_needed && + @latest_rs.rollout_data["availableReplicas"].to_i >= minimum_needed + end end end diff --git a/lib/kubernetes-deploy/kubernetes_resource/replica_set.rb b/lib/kubernetes-deploy/kubernetes_resource/replica_set.rb index 25a091ce7..560140344 100644 --- a/lib/kubernetes-deploy/kubernetes_resource/replica_set.rb +++ b/lib/kubernetes-deploy/kubernetes_resource/replica_set.rb @@ -3,7 +3,7 @@ module KubernetesDeploy class ReplicaSet < PodSetBase TIMEOUT = 5.minutes - attr_reader :desired_replicas, :pods + attr_reader :desired_replicas, :pods, :rollout_data def initialize(namespace:, context:, definition:, logger:, parent: nil, deploy_started_at: nil) @parent = parent diff --git a/test/fixtures/slow-cloud/web.yml.erb b/test/fixtures/slow-cloud/web.yml.erb index fa8dc6bc2..39077f94b 100644 --- a/test/fixtures/slow-cloud/web.yml.erb +++ b/test/fixtures/slow-cloud/web.yml.erb @@ -36,7 +36,7 @@ metadata: name: web annotations: shipit.shopify.io/restart: "true" - kubernetes-deploy.shopify.io/min-unavailable-rollout: 2 + kubernetes-deploy.shopify.io/required-rollout: maxUnavailable spec: replicas: 3 strategy: @@ -52,6 +52,11 @@ spec: spec: containers: - name: app + readinessProbe: + exec: + command: + - sleep + - '2' image: nginx:alpine ports: - containerPort: 80 diff --git a/test/integration/kubernetes_deploy_test.rb b/test/integration/kubernetes_deploy_test.rb index b82b0a01b..0fc3c5336 100644 --- a/test/integration/kubernetes_deploy_test.rb +++ b/test/integration/kubernetes_deploy_test.rb @@ -633,25 +633,15 @@ def test_can_deploy_deployment_with_zero_replicas end def test_can_deploy_deployment_with_partial_rollout_success - original_timeout = KubernetesDeploy::Deployment::TIMEOUT - begin - KubernetesDeploy::Deployment.const_set('TIMEOUT', 10.seconds) - result = deploy_fixtures("slow-cloud", subset: ["configmap-data.yml", "web.yml.erb"]) - assert_deploy_success(result) + result = deploy_fixtures("slow-cloud", subset: ["configmap-data.yml", "web.yml.erb"]) + assert_deploy_success(result) - result = deploy_fixtures("slow-cloud", subset: ["configmap-data.yml", "web.yml.erb"]) do |fixtures| - web = fixtures["web.yml.erb"]["Deployment"].first - probe = { 'readinessProbe' => { 'exec' => { 'command' => %w(sleep 4) } } } - web['spec']['template']['spec']['containers'].first.merge!(probe) - end + result = deploy_fixtures("slow-cloud", subset: ["configmap-data.yml", "web.yml.erb"]) + assert_deploy_success(result) - assert_deploy_success(result) - assert_logs_match_all( - [%r{Deployment\/web\s+[34] replicas, 3 updatedReplicas, 2 availableReplicas, [12] unavailableReplica}] - ) - ensure - KubernetesDeploy::Deployment.const_set('TIMEOUT', original_timeout) - end + assert_logs_match_all( + [%r{Deployment\/web\s+[34] replicas, 3 updatedReplicas, 2 availableReplicas, [12] unavailableReplica}] + ) end def test_deploy_aborts_immediately_if_metadata_name_missing @@ -847,7 +837,7 @@ def test_roll_back_a_bad_deploy container = fixtures["cannot_run.yml"]["Deployment"].first["spec"]["template"]["spec"]["containers"].first container["command"] = %w(sleep 8000) end - assert_deploy_success(result) + (result) all_rs = v1beta1_kubeclient.get_replica_sets(namespace: @namespace) assert_equal 2, all_rs.length, "Test premise failure: Rollback created a new RS" diff --git a/test/integration/restart_task_test.rb b/test/integration/restart_task_test.rb index c787a3d03..f34db114d 100644 --- a/test/integration/restart_task_test.rb +++ b/test/integration/restart_task_test.rb @@ -206,6 +206,27 @@ def test_restart_failure in_order: true) end + def test_can_deploy_deployment_with_partial_rollout_success + result = deploy_fixtures("slow-cloud", subset: ["configmap-data.yml", "web.yml.erb"]) + assert_deploy_success(result) + + restart = build_restart_task + assert_restart_success(restart.perform(["web"])) + + assert_logs_match_all([ + "Configured to restart deployments by name: web", + "Triggered `web` restart", + "Waiting for rollout", + %r{Successfully restarted in \d+\.\d+s: Deployment/web}, + "Result: SUCCESS", + "Successfully restarted 1 resource", + %r{Deployment\/web\s+[34] replicas, 3 updatedReplicas, 2 availableReplicas, [12] unavailableReplica} + ], + in_order: true) + + assert fetch_restarted_at("web"), "RESTARTED_AT is present after the restart" + end + private def build_restart_task