diff --git a/lib/kubernetes-deploy/kubernetes_resource/deployment.rb b/lib/kubernetes-deploy/kubernetes_resource/deployment.rb index a6028836b..6e67b7780 100644 --- a/lib/kubernetes-deploy/kubernetes_resource/deployment.rb +++ b/lib/kubernetes-deploy/kubernetes_resource/deployment.rb @@ -63,8 +63,7 @@ def deploy_succeeded? @latest_rs.ready_replicas >= minimum_needed && @latest_rs.available_replicas >= minimum_needed else - raise "#{REQUIRED_ROLLOUT_ANNOTATION}:#{required_rollout} is invalid "\ - " Acceptable options: #{REQUIRED_ROLLOUT_TYPES.join(',')}" + raise FatalDeploymentError, rollout_annotation_err_msg end end @@ -104,14 +103,13 @@ def validate_definition super unless REQUIRED_ROLLOUT_TYPES.include?(required_rollout) - @validation_errors << "#{REQUIRED_ROLLOUT_ANNOTATION}:#{required_rollout} is invalid "\ - "Acceptable options: #{REQUIRED_ROLLOUT_TYPES.join(',')}" + @validation_errors << rollout_annotation_err_msg end - if required_rollout.downcase == 'maxunavailable' && @definition.dig('spec', 'strategy').respond_to?(:downcase) && - @definition.dig('spec', 'strategy').downcase == 'recreate' - @validation_errors << "#{REQUIRED_ROLLOUT_ANNOTATION}:#{required_rollout} is invalid "\ - "with strategy 'rollingUpdate'" + strategy = @definition.dig('spec', 'strategy', 'type').to_s + if required_rollout.downcase == 'maxunavailable' && strategy.downcase != 'rollingupdate' + @validation_errors << "'#{REQUIRED_ROLLOUT_ANNOTATION}: #{required_rollout}' is incompatible "\ + "with strategy '#{strategy}'" end @validation_errors.empty? @@ -119,6 +117,11 @@ def validate_definition private + def rollout_annotation_err_msg + "'#{REQUIRED_ROLLOUT_ANNOTATION}: #{required_rollout}' is invalid. "\ + "Acceptable values: #{REQUIRED_ROLLOUT_TYPES.join(', ')}" + end + def deploy_failing_to_progress? return false unless @progress_condition.present? diff --git a/test/fixtures/for_unit_tests/deployment_test.yml b/test/fixtures/for_unit_tests/deployment_test.yml new file mode 100644 index 000000000..b262abdce --- /dev/null +++ b/test/fixtures/for_unit_tests/deployment_test.yml @@ -0,0 +1,62 @@ +--- +apiVersion: apps/v1beta1 +kind: Deployment +metadata: + name: web + uid: foobar + annotations: + "deployment.kubernetes.io/revision": "1" +spec: + replicas: 3 + progressDeadlineSeconds: 10 + strategy: + type: RollingUpdate + rollingUpdate: + maxSurge: 1 + maxUnavailable: 1 + selector: + matchLabels: + name: web + app: hello-cloud + template: + metadata: + labels: + name: web + app: hello-cloud + spec: + containers: + - name: app + image: busybox +status: + replicas: 3 + conditions: + - type: Progressing + status: True + lastUpdateTime: "2018-01-09 22:56:45 UTC" + +--- +apiVersion: apps/v1beta1 +kind: ReplicaSet +metadata: + name: web-1 + annotations: + "deployment.kubernetes.io/revision": "1" + ownerReferences: + - uid: foobar +spec: + replicas: 3 + selector: + matchLabels: + name: web + app: hello-cloud + template: + metadata: + labels: + name: web + app: hello-cloud + spec: + containers: + - name: app + image: busybox +status: + replicas: 3 diff --git a/test/unit/kubernetes-deploy/kubernetes_resource/deployment_test.rb b/test/unit/kubernetes-deploy/kubernetes_resource/deployment_test.rb index 94164460e..082a606ea 100644 --- a/test/unit/kubernetes-deploy/kubernetes_resource/deployment_test.rb +++ b/test/unit/kubernetes-deploy/kubernetes_resource/deployment_test.rb @@ -2,118 +2,332 @@ require 'test_helper' class DeploymentTest < KubernetesDeploy::TestCase + def setup + KubernetesDeploy::Kubectl.any_instance.expects(:run).never + super + end + def test_deploy_succeeded_with_none_annotation - rollout = { - 'metadata' => { - 'name' => 'fake', - 'annotations' => { KubernetesDeploy::Deployment::REQUIRED_ROLLOUT_ANNOTATION => 'none' } - } + deployment_status = { + "replicas" => 3, + "updatedReplicas" => 1, + "unavailableReplicas" => 1, + "availableReplicas" => 0 } - deploy = KubernetesDeploy::Deployment.new(namespace: "", context: "", logger: logger, definition: rollout) - deploy.instance_variable_set(:@latest_rs, true) - + rs_status = { + "replicas" => 3, + "availableReplicas" => 0, + "readyReplicas" => 0 + } + dep_template = build_deployment_template(status: deployment_status, rollout: 'none', + strategy: 'RollingUpdate', max_unavailable: 1) + deploy = build_synced_deployment(template: dep_template, replica_sets: [build_rs_template(status: rs_status)]) assert deploy.deploy_succeeded? end + def test_deploy_succeeded_is_false_with_none_annotation_before_new_rs_created + deployment_status = { + "replicas" => 3, + "updatedReplicas" => 3, + "unavailableReplicas" => 0, + "availableReplicas" => 3 + } + deploy = build_synced_deployment( + template: build_deployment_template(status: deployment_status, rollout: 'none'), + replica_sets: [] + ) + refute deploy.deploy_succeeded? + end + def test_deploy_succeeded_with_max_unavailable - rollout = { - 'metadata' => { - 'name' => 'fake', - 'annotations' => { KubernetesDeploy::Deployment::REQUIRED_ROLLOUT_ANNOTATION => 'maxUnavailable' } - } + deployment_status = { + "replicas" => 3, # one terminating in old rs, one starting in new rs, one up in new rs + "updatedReplicas" => 2, + "unavailableReplicas" => 2, + "availableReplicas" => 1 + } + + rs_status = { + "replicas" => 2, + "availableReplicas" => 1, + "readyReplicas" => 1 } + replica_sets = [build_rs_template(status: rs_status)] - deploy = KubernetesDeploy::Deployment.new(namespace: "", context: "", logger: logger, definition: rollout) - mock_rs = Minitest::Mock.new - needed = 2 - mock_rs.expect :present?, true - mock_rs.expect :desired_replicas, needed - mock_rs.expect :ready_replicas, needed - mock_rs.expect :available_replicas, needed - deploy.instance_variable_set(:@max_unavailable, 0) - deploy.instance_variable_set(:@latest_rs, mock_rs) - deploy.instance_variable_set(:@desired_replicas, needed) + deploy = build_synced_deployment( + template: build_deployment_template(status: deployment_status, rollout: 'maxUnavailable', max_unavailable: 3), + replica_sets: replica_sets + ) + assert deploy.deploy_succeeded? + deploy = build_synced_deployment( + template: build_deployment_template(status: deployment_status, rollout: 'maxUnavailable', max_unavailable: 2), + replica_sets: replica_sets + ) assert deploy.deploy_succeeded? + + deploy = build_synced_deployment( + template: build_deployment_template(status: deployment_status, rollout: 'maxUnavailable', max_unavailable: 1), + replica_sets: replica_sets + ) + refute deploy.deploy_succeeded? + + deploy = build_synced_deployment( + template: build_deployment_template(status: deployment_status, rollout: 'maxUnavailable', max_unavailable: 0), + replica_sets: replica_sets + ) + refute deploy.deploy_succeeded? end - def test_deploy_succeeded_fails_with_max_unavailable - rollout = { - 'metadata' => { - 'name' => 'fake', - 'annotations' => { KubernetesDeploy::Deployment::REQUIRED_ROLLOUT_ANNOTATION => 'maxUnavailable' } - } + def test_deploy_succeeded_with_max_unavailable_as_percent + deployment_status = { + "replicas" => 3, + "updatedReplicas" => 2, + "unavailableReplicas" => 2, + "availableReplicas" => 1 + } + + rs_status = { + "replicas" => 2, + "availableReplicas" => 1, + "readyReplicas" => 1 } + replica_sets = [build_rs_template(status: rs_status)] + + dep_template = build_deployment_template(status: deployment_status, + rollout: 'maxUnavailable', max_unavailable: '100%') + deploy = build_synced_deployment(template: dep_template, replica_sets: replica_sets) + assert deploy.deploy_succeeded? + + # rounds up to two max + deploy = build_synced_deployment( + template: build_deployment_template(status: deployment_status, rollout: 'maxUnavailable', max_unavailable: '67%'), + replica_sets: replica_sets + ) + assert deploy.deploy_succeeded? - deploy = KubernetesDeploy::Deployment.new(namespace: "", context: "", logger: logger, definition: rollout) - mock_rs = Minitest::Mock.new - needed = 2 - mock_rs.expect :present?, true - mock_rs.expect :desired_replicas, needed - mock_rs.expect :ready_replicas, needed - 1 - mock_rs.expect :available_replicas, needed - 1 - deploy.instance_variable_set(:@max_unavailable, 0) - deploy.instance_variable_set(:@latest_rs, mock_rs) - deploy.instance_variable_set(:@desired_replicas, needed) + # rounds down to one max + deploy = build_synced_deployment( + template: build_deployment_template(status: deployment_status, rollout: 'maxUnavailable', max_unavailable: '66%'), + replica_sets: replica_sets + ) + refute deploy.deploy_succeeded? + deploy = build_synced_deployment( + template: build_deployment_template(status: deployment_status, rollout: 'maxUnavailable', max_unavailable: '0%'), + replica_sets: replica_sets + ) refute deploy.deploy_succeeded? end - def test_deploy_succeeded_fails_with_max_unavailable_as_a_percent - rollout = { - 'metadata' => { - 'name' => 'fake', - 'annotations' => { KubernetesDeploy::Deployment::REQUIRED_ROLLOUT_ANNOTATION => 'maxUnavailable' } - } - } + def test_deploy_succeeded_raises_with_invalid_rollout_annotation + deploy = build_synced_deployment( + template: build_deployment_template(rollout: 'bad'), + replica_sets: [build_rs_template] + ) + msg = "'#{KubernetesDeploy::Deployment::REQUIRED_ROLLOUT_ANNOTATION}: bad' is "\ + "invalid. Acceptable values: #{KubernetesDeploy::Deployment::REQUIRED_ROLLOUT_TYPES.join(', ')}" + assert_raises_message(KubernetesDeploy::FatalDeploymentError, msg) do + deploy.deploy_succeeded? + end + end + + def test_validation_fails_with_invalid_rollout_annotation + deploy = build_synced_deployment(template: build_deployment_template(rollout: 'bad'), replica_sets: []) + deploy.kubectl.expects(:run).with('create', '-f', anything, '--dry-run', '--output=name', anything).returns( + ["", "super failed", SystemExit.new(1)] + ) + refute deploy.validate_definition + + expected = <<~STRING.strip + super failed + '#{KubernetesDeploy::Deployment::REQUIRED_ROLLOUT_ANNOTATION}: bad' is invalid. Acceptable values: #{KubernetesDeploy::Deployment::REQUIRED_ROLLOUT_TYPES.join(', ')} + STRING + assert_equal expected, deploy.validation_error_msg + end - deploy = KubernetesDeploy::Deployment.new(namespace: "", context: "", logger: logger, definition: rollout) - mock_rs = Minitest::Mock.new - needed = 2 - mock_rs.expect :present?, true - mock_rs.expect :desired_replicas, needed - mock_rs.expect :ready_replicas, needed - 1 - mock_rs.expect :available_replicas, needed - 1 - deploy.instance_variable_set(:@max_unavailable, '49%') - deploy.instance_variable_set(:@latest_rs, mock_rs) - deploy.instance_variable_set(:@desired_replicas, needed) + def test_validation_fails_with_invalid_mix_of_annotation + deploy = build_synced_deployment( + template: build_deployment_template(rollout: 'maxUnavailable', strategy: 'Recreate'), + replica_sets: [build_rs_template] + ) + deploy.kubectl.expects(:run).with('create', '-f', anything, '--dry-run', '--output=name', anything).returns( + ["", "super failed", SystemExit.new(1)] + ) + refute deploy.validate_definition + expected = <<~STRING.strip + super failed + '#{KubernetesDeploy::Deployment::REQUIRED_ROLLOUT_ANNOTATION}: maxUnavailable' is incompatible with strategy 'Recreate' + STRING + assert_equal expected, deploy.validation_error_msg + end + + def test_deploy_succeeded_not_fooled_by_stale_rs_data_in_deploy_status + deployment_status = { + "replicas" => 3, + "updatedReplicas" => 3, # stale -- hasn't been updated since new RS was created + "unavailableReplicas" => 0, + "availableReplicas" => 3 + } + + rs_status = { + "replicas" => 1, + "availableReplicas" => 0, + "readyReplicas" => 0 + } + deploy = build_synced_deployment( + template: build_deployment_template(status: deployment_status, rollout: 'full', max_unavailable: 1), + replica_sets: [build_rs_template(status: rs_status)] + ) refute deploy.deploy_succeeded? end - def test_deploy_succeeded_raises_with_invalid_annotation - rollout = { - 'metadata' => { - 'name' => 'fake', - 'annotations' => { KubernetesDeploy::Deployment::REQUIRED_ROLLOUT_ANNOTATION => 'invalid' } + def test_deploy_timed_out_with_hard_timeout + Timecop.freeze do + deploy = build_synced_deployment( + template: build_deployment_template(status: { "replicas" => 3, "conditions" => [] }), + replica_sets: [build_rs_template(status: { "replica" => 1 })] + ) + deploy.deploy_started_at = Time.now.utc - KubernetesDeploy::Deployment::TIMEOUT + refute deploy.deploy_timed_out? + + deploy.deploy_started_at = Time.now.utc - KubernetesDeploy::Deployment::TIMEOUT - 1 + assert deploy.deploy_timed_out? + assert_equal "Timeout reason: hard deadline for Deployment\nLatest ReplicaSet: web-1", + deploy.timeout_message.strip + end + end + + def test_deploy_timed_out_based_on_progress_deadline + Timecop.freeze do + deployment_status = { + "replicas" => 3, + "conditions" => [{ + "type" => "Progressing", + "status" => 'False', + "lastUpdateTime" => Time.now.utc - 10.seconds, + "reason" => "Failed to progress" + }] } - } + deploy = build_synced_deployment( + template: build_deployment_template(status: deployment_status), + replica_sets: [build_rs_template(status: { "replica" => 1 })] + ) + deploy.deploy_started_at = Time.now.utc - 3.minutes + deploy.kubectl.expects(:server_version).returns(Gem::Version.new("1.8")) + + assert deploy.deploy_timed_out? + assert_equal "Timeout reason: Failed to progress\nLatest ReplicaSet: web-1", deploy.timeout_message.strip + end + end + + def test_deploy_timed_out_based_on_progress_deadline_ignores_conditions_older_than_the_deploy + Timecop.freeze do + deployment_status = { + "replicas" => 3, + "conditions" => [{ + "type" => "Progressing", + "status" => 'False', + "lastUpdateTime" => Time.now.utc - 10.seconds, + "reason" => "Failed to progress" + }] + } + deploy = build_synced_deployment( + template: build_deployment_template(status: deployment_status), + replica_sets: [build_rs_template(status: { "replica" => 1 })] + ) + deploy.kubectl.expects(:server_version).returns(Gem::Version.new("1.8")).at_least_once + + deploy.deploy_started_at = nil # not started yet + refute deploy.deploy_timed_out? - deploy = KubernetesDeploy::Deployment.new(namespace: "foo", context: "", logger: logger, definition: rollout) - deploy.instance_variable_set(:@latest_rs, true) + deploy.deploy_started_at = Time.now.utc - 4.seconds # 10s ago is before deploy started + refute deploy.deploy_timed_out? - assert_raises(RuntimeError) { deploy.deploy_succeeded? } + deploy.deploy_started_at = Time.now.utc - 5.seconds # 10s ago is "equal" to deploy time (fudge for clock skew) + assert deploy.deploy_timed_out? + end end - def test_deploy_succeeded_raises_with_invalid_mix_of_annotation - rollout = { - 'spec' => { - 'strategy' => 'recreate' - }, - 'metadata' => { - 'name' => 'fake', - 'annotations' => { KubernetesDeploy::Deployment::REQUIRED_ROLLOUT_ANNOTATION => 'maxUnavailable' } + def test_deploy_timed_out_based_on_progress_deadline_accommodates_stale_conditions_bug_in_k8s_176_and_lower + Timecop.freeze do + deployment_status = { + "replicas" => 3, + "conditions" => [{ + "type" => "Progressing", + "status" => 'False', + "lastUpdateTime" => Time.now.utc - 5.seconds, + "reason" => "Failed to progress" + }] } - } + deploy = build_synced_deployment( + template: build_deployment_template(status: deployment_status), + replica_sets: [build_rs_template(status: { "replica" => 1 })] + ) + deploy.deploy_started_at = Time.now.utc - 5.seconds # progress deadline of 10s has not elapsed + deploy.kubectl.expects(:server_version).returns(Gem::Version.new("1.7.6")) + + refute deploy.deploy_timed_out? + end + end - kubectl_mock = Minitest::Mock.new - status_mock = Minitest::Mock.new - status_mock.expect :success?, true - kubectl_mock.expect(:run, [true, true, status_mock], [Object, Object, Object, Object, Object, Object]) - deploy = KubernetesDeploy::Deployment.new(namespace: "", context: "", logger: logger, definition: rollout) - deploy.instance_variable_set(:@kubectl, kubectl_mock) + private - refute deploy.validate_definition + def build_deployment_template(status: { 'replicas' => 3 }, rollout: nil, + strategy: 'rollingUpdate', max_unavailable: nil) + + base_deployment_manifest = fixtures.find { |fixture| fixture["kind"] == "Deployment" } + result = base_deployment_manifest.deep_merge("status" => status) + result["metadata"]["annotations"][KubernetesDeploy::Deployment::REQUIRED_ROLLOUT_ANNOTATION] = rollout if rollout + + if spec_override = status["replicas"].presence # ignores possibility of surge; need a spec_replicas arg for that + result["spec"]["replicas"] = spec_override + end + + if strategy == "Recreate" + result["spec"]["strategy"] = { "type" => strategy } + end + + if max_unavailable + result["spec"]["strategy"]["rollingUpdate"] = { "maxUnavailable" => max_unavailable } + end + + result + end + + def build_rs_template(status: { 'replicas' => 3 }) + base_rs_manifest = fixtures.find { |fixture| fixture["kind"] == "ReplicaSet" } + result = base_rs_manifest.deep_merge("status" => status) + + if spec_override = status["replicas"].presence # ignores possibility of surge; need a spec_replicas arg for that + result["spec"]["replicas"] = spec_override + end + result + end + + def build_synced_deployment(template:, replica_sets:) + deploy = KubernetesDeploy::Deployment.new(namespace: "test", context: "nope", logger: logger, definition: template) + deploy.kubectl.expects(:run).with("get", "Deployment", "web", "--output=json").returns( + [template.to_json, "", SystemExit.new(0)] + ) + + if replica_sets.present? + KubernetesDeploy::ReplicaSet.any_instance.expects(:kubectl).returns(deploy.kubectl) + deploy.kubectl.expects(:run).with("get", "pods", "-a", "--output=json", anything).returns( + ['{ "items": [] }', "", SystemExit.new(0)] + ) + end + + deploy.kubectl.expects(:run).with("get", "replicasets", "--output=json", anything).returns( + [{ "items" => replica_sets }.to_json, "", SystemExit.new(0)] + ) + deploy.sync + deploy + end + + def fixtures + @fixtures ||= YAML.load_stream(File.read(File.join(fixture_path('for_unit_tests'), 'deployment_test.yml'))) end end