Skip to content

Commit

Permalink
add a restart test
Browse files Browse the repository at this point in the history
  • Loading branch information
dturn committed Dec 14, 2017
1 parent 2846716 commit 6b7bbd8
Show file tree
Hide file tree
Showing 5 changed files with 66 additions and 45 deletions.
55 changes: 30 additions & 25 deletions lib/kubernetes-deploy/kubernetes_resource/deployment.rb
Original file line number Diff line number Diff line change
Expand Up @@ -111,31 +111,6 @@ def running_rs
end
end

def min_unavailable_rollout
@definition.dig('metadata', 'annotations', 'kubernetes-deploy.shopify.io/min-unavailable-rollout')
end

def minimum_unavailable_replicas_to_succeeded
desired = @desired_replicas

if min_unavailable_rollout =~ /%/
desired *= (100 - min_unavailable_rollout.to_i) / 100.0
else
desired = min_unavailable_rollout.to_i
end

desired.to_i
end

def partial_deployment_success
minimum_needed = minimum_unavailable_replicas_to_succeeded

running_rs.size <= 2 &&
@latest_rs.desired_replicas > minimum_needed &&
@rollout_data["updatedReplicas"].to_i >= minimum_needed &&
@rollout_data["availableReplicas"].to_i >= minimum_needed
end

def find_latest_rs
current_revision = @deployment_data["metadata"]["annotations"]["deployment.kubernetes.io/revision"]

Expand All @@ -157,5 +132,35 @@ def find_latest_rs
rs.sync(latest_rs_data)
rs
end

def min_unavailable_rollout
@definition.dig('metadata', 'annotations', 'kubernetes-deploy.shopify.io/required-rollout')
end

def minimum_unavailable_replicas_to_succeeded
desired = @desired_replicas

case min_unavailable_rollout
when 'maxUnavailable'
max_unavailable = @definition.dig('spec', 'strategy', 'rollingUpdate', 'maxUnavailable')
if max_unavailable =~ /%/
(desired * (100 - max_unavailable.to_i) / 100.0).to_i
else
desired - max_unavailable.to_i
end
when 'none'
0
else
raise "#{min_unavailable_rollout} is not a valid value for required-rollout"
end
end

def partial_deployment_success
minimum_needed = minimum_unavailable_replicas_to_succeeded

@latest_rs.desired_replicas >= minimum_needed &&
@latest_rs.rollout_data["readyReplicas"].to_i >= minimum_needed &&
@latest_rs.rollout_data["availableReplicas"].to_i >= minimum_needed
end
end
end
2 changes: 1 addition & 1 deletion lib/kubernetes-deploy/kubernetes_resource/replica_set.rb
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
module KubernetesDeploy
class ReplicaSet < PodSetBase
TIMEOUT = 5.minutes
attr_reader :desired_replicas, :pods
attr_reader :desired_replicas, :pods, :rollout_data

def initialize(namespace:, context:, definition:, logger:, parent: nil, deploy_started_at: nil)
@parent = parent
Expand Down
7 changes: 6 additions & 1 deletion test/fixtures/slow-cloud/web.yml.erb
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@ metadata:
name: web
annotations:
shipit.shopify.io/restart: "true"
kubernetes-deploy.shopify.io/min-unavailable-rollout: 2
kubernetes-deploy.shopify.io/required-rollout: maxUnavailable
spec:
replicas: 3
strategy:
Expand All @@ -52,6 +52,11 @@ spec:
spec:
containers:
- name: app
readinessProbe:
exec:
command:
- sleep
- '2'
image: nginx:alpine
ports:
- containerPort: 80
Expand Down
26 changes: 8 additions & 18 deletions test/integration/kubernetes_deploy_test.rb
Original file line number Diff line number Diff line change
Expand Up @@ -633,25 +633,15 @@ def test_can_deploy_deployment_with_zero_replicas
end

def test_can_deploy_deployment_with_partial_rollout_success
original_timeout = KubernetesDeploy::Deployment::TIMEOUT
begin
KubernetesDeploy::Deployment.const_set('TIMEOUT', 10.seconds)
result = deploy_fixtures("slow-cloud", subset: ["configmap-data.yml", "web.yml.erb"])
assert_deploy_success(result)
result = deploy_fixtures("slow-cloud", subset: ["configmap-data.yml", "web.yml.erb"])
assert_deploy_success(result)

result = deploy_fixtures("slow-cloud", subset: ["configmap-data.yml", "web.yml.erb"]) do |fixtures|
web = fixtures["web.yml.erb"]["Deployment"].first
probe = { 'readinessProbe' => { 'exec' => { 'command' => %w(sleep 4) } } }
web['spec']['template']['spec']['containers'].first.merge!(probe)
end
result = deploy_fixtures("slow-cloud", subset: ["configmap-data.yml", "web.yml.erb"])
assert_deploy_success(result)

assert_deploy_success(result)
assert_logs_match_all(
[%r{Deployment\/web\s+[34] replicas, 3 updatedReplicas, 2 availableReplicas, [12] unavailableReplica}]
)
ensure
KubernetesDeploy::Deployment.const_set('TIMEOUT', original_timeout)
end
assert_logs_match_all(
[%r{Deployment\/web\s+[34] replicas, 3 updatedReplicas, 2 availableReplicas, [12] unavailableReplica}]
)
end

def test_deploy_aborts_immediately_if_metadata_name_missing
Expand Down Expand Up @@ -847,7 +837,7 @@ def test_roll_back_a_bad_deploy
container = fixtures["cannot_run.yml"]["Deployment"].first["spec"]["template"]["spec"]["containers"].first
container["command"] = %w(sleep 8000)
end
assert_deploy_success(result)
(result)

all_rs = v1beta1_kubeclient.get_replica_sets(namespace: @namespace)
assert_equal 2, all_rs.length, "Test premise failure: Rollback created a new RS"
Expand Down
21 changes: 21 additions & 0 deletions test/integration/restart_task_test.rb
Original file line number Diff line number Diff line change
Expand Up @@ -206,6 +206,27 @@ def test_restart_failure
in_order: true)
end

def test_can_deploy_deployment_with_partial_rollout_success
result = deploy_fixtures("slow-cloud", subset: ["configmap-data.yml", "web.yml.erb"])
assert_deploy_success(result)

restart = build_restart_task
assert_restart_success(restart.perform(["web"]))

assert_logs_match_all([
"Configured to restart deployments by name: web",
"Triggered `web` restart",
"Waiting for rollout",
%r{Successfully restarted in \d+\.\d+s: Deployment/web},
"Result: SUCCESS",
"Successfully restarted 1 resource",
%r{Deployment\/web\s+[34] replicas, 3 updatedReplicas, 2 availableReplicas, [12] unavailableReplica}
],
in_order: true)

assert fetch_restarted_at("web"), "RESTARTED_AT is present after the restart"
end

private

def build_restart_task
Expand Down

0 comments on commit 6b7bbd8

Please sign in to comment.