diff --git a/dev.yml b/dev.yml index e0f07e335..b3c71d40d 100644 --- a/dev.yml +++ b/dev.yml @@ -8,7 +8,7 @@ up: - custom: name: Minikube Cluster met?: test $(minikube status | grep Running | wc -l) -eq 2 && $(minikube status | grep -q 'Correctly Configured') - meet: minikube start --vm-driver=xhyve --kubernetes-version=v1.7.5 + meet: minikube start --vm-driver=xhyve --kubernetes-version=v1.8.0 down: minikube stop commands: reset-minikube: minikube delete && rm -rf ~/.minikube diff --git a/kubernetes-deploy.gemspec b/kubernetes-deploy.gemspec index c01dbd3bd..968f72d6b 100644 --- a/kubernetes-deploy.gemspec +++ b/kubernetes-deploy.gemspec @@ -24,12 +24,13 @@ Gem::Specification.new do |spec| spec.required_ruby_version = '>= 2.3.0' spec.add_dependency "activesupport", ">= 4.2" - spec.add_dependency "kubeclient", "~> 2.4" + spec.add_dependency "kubeclient", "~> 2.5.1" spec.add_dependency "rest-client", ">= 1.7" # Minimum required by kubeclient. Remove when kubeclient releases v3.0. spec.add_dependency "googleauth", ">= 0.5" spec.add_dependency "ejson", "1.0.1" spec.add_dependency "colorize", "~> 0.8" spec.add_dependency "statsd-instrument", "~> 2.1" + spec.add_dependency "jsonpath", "0.8.8" spec.add_development_dependency "bundler" spec.add_development_dependency "rake", "~> 10.0" diff --git a/lib/kubernetes-deploy/deploy_task.rb b/lib/kubernetes-deploy/deploy_task.rb index e15507cb7..47792db80 100644 --- a/lib/kubernetes-deploy/deploy_task.rb +++ b/lib/kubernetes-deploy/deploy_task.rb @@ -5,7 +5,9 @@ require 'yaml' require 'shellwords' require 'tempfile' +require 'kubernetes-deploy/discoverable_resource' require 'kubernetes-deploy/kubernetes_resource' + %w( cloudsql config_map @@ -29,6 +31,7 @@ bucket stateful_set cron_job + customresourcedefinition ).each do |subresource| require "kubernetes-deploy/kubernetes_resource/#{subresource}" end @@ -41,17 +44,6 @@ module KubernetesDeploy class DeployTask include KubeclientBuilder - PREDEPLOY_SEQUENCE = %w( - ResourceQuota - Cloudsql - Redis - Memcached - Bugsnag - ConfigMap - PersistentVolumeClaim - ServiceAccount - Pod - ) PROTECTED_NAMESPACES = %w( default kube-system @@ -66,23 +58,18 @@ class DeployTask # core/v1/ReplicationController -- superseded by deployments/replicasets # extensions/v1beta1/ReplicaSet -- managed by deployments # core/v1/Secret -- should not committed / managed by shipit - def prune_whitelist - wl = %w( - core/v1/ConfigMap - core/v1/Pod - core/v1/Service - batch/v1/Job - extensions/v1beta1/DaemonSet - extensions/v1beta1/Deployment - apps/v1beta1/Deployment - extensions/v1beta1/Ingress - apps/v1beta1/StatefulSet - autoscaling/v1/HorizontalPodAutoscaler - ) - if server_version >= Gem::Version.new('1.8.0') - wl << "batch/v1beta1/CronJob" - end - wl + def prune_whitelist + prunable_resources = all_resources.select(&:prunable?) + prunable_resources.map(&:qualified_kind) + end + + def predeploy_sequence + predeploy_resources = all_resources.select(&:predeploy?) + predeploy_resources.map(&:kind) # Predeploy list does not use fully-qualifed kind + end + + def all_resources + DiscoverableResource.all + KubernetesResource.all end def server_version @@ -111,7 +98,8 @@ def run(verify_result: true, allow_protected_ns: false, prune: true) validate_configuration(allow_protected_ns: allow_protected_ns, prune: prune) confirm_context_exists confirm_namespace_exists - resources = discover_resources + discover_resources + resources = load_resource_from_file validate_definitions(resources) @logger.phase_heading("Checking initial resource statuses") @@ -186,12 +174,12 @@ def find_bad_files_from_kubectl_output(stderr) end def deploy_has_priority_resources?(resources) - resources.any? { |r| PREDEPLOY_SEQUENCE.include?(r.type) } + resources.any? { |r| predeploy_sequence.include?(r.kind) } end def predeploy_priority_resources(resource_list) - PREDEPLOY_SEQUENCE.each do |resource_type| - matching_resources = resource_list.select { |r| r.type == resource_type } + predeploy_sequence.each do |resource_kind| + matching_resources = resource_list.select { |r| r.kind == resource_kind } next if matching_resources.empty? deploy_resources(matching_resources, verify: true, record_summary: false) @@ -217,6 +205,10 @@ def validate_definitions(resources) end def discover_resources + DiscoverableResource.discover(context: @context, logger: @logger, server_version: server_version) + end + + def load_resource_from_file resources = [] @logger.info("Discovering templates:") @@ -224,7 +216,7 @@ def discover_resources next unless filename.end_with?(".yml.erb", ".yml", ".yaml", ".yaml.erb") split_templates(filename) do |r_def| - r = KubernetesResource.build(namespace: @namespace, context: @context, logger: @logger, definition: r_def) + r = DiscoverableResource.build(namespace: @namespace, context: @context, logger: @logger, definition: r_def) resources << r @logger.info " - #{r.id}" end @@ -394,7 +386,7 @@ def apply_all(resources, prune) if prune command.push("--prune", "--all") - prune_whitelist.each { |type| command.push("--prune-whitelist=#{type}") } + prune_whitelist.each { |kind| command.push("--prune-whitelist=#{kind}") } end out, err, st = kubectl.run(*command, log_failure: false) diff --git a/lib/kubernetes-deploy/discoverable_resource.rb b/lib/kubernetes-deploy/discoverable_resource.rb new file mode 100644 index 000000000..c3272cc19 --- /dev/null +++ b/lib/kubernetes-deploy/discoverable_resource.rb @@ -0,0 +1,208 @@ +# frozen_string_literal: true +require 'kubernetes-deploy/kubernetes_resource' +require 'kubernetes-deploy/kubeclient_builder' +require 'erb' +require 'json' +require "jsonpath" + +module KubernetesDeploy + class DiscoverableResource < KubernetesResource + extend KubernetesDeploy::KubeclientBuilder + + TRUE_VALUES = [true, 1, '1', 't', 'T', 'true', 'TRUE'].to_set + DEPLOY_METADATA_ANNOTATION = 'kubernetes-deploy.shopify.io/metadata' + + def self.child_classes + @child_classes ||= Set.new + end + + def self.discover(context:, logger:, server_version:) + logger.info("Discovering custom resources:") + with_retries { discover_groups(context) } + if server_version >= Gem::Version.new('1.7.0') + kube_client = v1beta1_crd_kubeclient(context) + with_retries { discover_crd(v1beta1_crd_kubeclient(context)) } + end + end + + def self.discover_groups(context) + kinds = discover_kinds(context) + kinds.each_pair do |key, val| + klass = get_static_class(kind: key) + next unless klass + klass.const_set(:GROUP, val[:group]) unless klass.constants.include?(:GROUP) + klass.const_set(:VERSION, val[:version]) unless klass.constants.include?(:VERSION) + end + end + + def self.discover_kinds(context) + kinds = {} + + # At the top level there is the core group (everything below /api/v1), + rest_client = v1_kubeclient(context).create_rest_client + raw_json = rest_client['v1'].get(rest_client.headers) + resource_list = JSON.parse(raw_json) + v1_group_version = { group: 'core', version: 'v1' } + resource_list['resources'].map do |res| + kind = res['kind'] + kinds[kind] = v1_group_version + end + + # ...and the named groups (at path /apis/$NAME/$VERSION) + rest_client = apis_kubeclient(context).create_rest_client + raw_json = rest_client.get(rest_client.headers) + group_list = JSON.parse(raw_json) + group_versions = group_list['groups'] + + # Map out all detected kinds to their (preferred) group version + group_versions.each do |group_version| + preferred_version = group_version['preferredVersion']['groupVersion'] + all_versions = group_version['versions'].map { |version| version['groupVersion']} + # Make sure the preferred version gets checked first. + all_versions.delete(preferred_version) + all_versions.unshift(preferred_version) + + # Grab kinds from all versions + all_versions.each do |group_version| + raw_response = rest_client[group_version].get(rest_client.headers) + json_response = JSON.parse(raw_response) + resources = json_response['resources'] + resources.each do |res| + kind = res['kind'] + next if kinds.has_key?(kind) # Respect the preferred version + group, _, version = group_version.rpartition('/') + kinds[kind] = { group: group, version: version } + end + end + end + + kinds + end + + def self.build(namespace:, context:, definition:, logger:) + opts = { namespace: namespace, context: context, definition: definition, logger: logger } + kind = definition["kind"] + group, _, version = definition['apiVersion'].rpartition('/') + + klass = get_static_class(kind: kind) + klass = get_dynamic_class(group: group, version: version, kind: kind) unless klass + klass.new(**opts) + end + + def self.get_static_class(kind:) + KubernetesDeploy.const_get(kind) if KubernetesDeploy.const_defined?(kind) + end + + def self.get_dynamic_class(group:, version:, kind:) + unless DiscoverableResource.const_defined?(kind) + generate_resource(group: group, version: version, kind: kind, annotations: {}) + end + DiscoverableResource.const_get(kind) + end + + def self.with_retries(retries=3, backoff=10) + yield + rescue KubeException => err + if (retries -= 1) > 0 + logger.warn("Retrying to discover CustomResourceDefinitions: #{err}") + sleep(backoff) + retry + else + logger.warn("Unable to discover CustomResourceDefinitions: #{err}") + end + end + + def self.discover_crd(client) + @child_classes = Set.new + resources = client.get_custom_resource_definitions + resources.each do |res| + kind = res.spec.names.kind + # Remove and redefine the class if it already exists so we can be up to date. + if DiscoverableResource.const_defined?(kind) + klass = DiscoverableResource.const_get(kind) + DiscoverableResource.send(:remove_const, kind) + child_classes.delete(klass) + end + generate_resource(group: res.spec.group, + version: res.spec.version, + kind: kind, + annotations: res.metadata.annotations) + end + end + + def self.generate_resource(group:, kind:, version:, annotations:) + deploy_metadata = annotations[DEPLOY_METADATA_ANNOTATION] || '{}' + metadata = JSON.parse(deploy_metadata) + raise FatalDeploymentError, "Invalid metadata content: #{metadata}" unless metadata.is_a?(Hash) + + prunable = parse_bool(metadata['prunable']) + predeploy = parse_bool(metadata['predeploy']) + + status_field = metadata['status-field'] + success_status = metadata['status-success'] + + resource_template = ERB.new <<-CLASS + class #{kind.capitalize} < DiscoverableResource + GROUP = '#{group}' + VERSION = '#{version}' + PREDEPLOY = #{predeploy} + PRUNABLE = #{prunable} + + <% if status_field && success_status %> + def deploy_succeeded? + getter = "get_#{kind.downcase}" + @client ||= DiscoverableResource.kubeclient(context: @context, resource_class: self.class) + raw_json = @client.send(getter, @name, @namespace, as: :raw) + query_path = JsonPath.new('#{status_field}') + current_status = query_path.first(raw_json) + current_status == '#{success_status}' + end + <% end %> + + self + end + CLASS + + rendered_template = resource_template.result(binding) + resource_class = self.class_eval(rendered_template) + end + + def self.parse_bool(value) + return true if TRUE_VALUES.include?(value) + false + end + + def self.kubeclient(context:, resource_class:) + _build_kubeclient( + api_version: resource_class.version, + context: context, + endpoint_path: "/apis/#{resource_class.group}" + ) + end + + def self.apis_kubeclient(context) + @apis_kubeclient ||= _build_kubeclient( + api_version: '', # The apis endpoint is not versioned + context: context, + endpoint_path: "/apis", + discover: false # Will fail on apis endpoint + ) + end + + def self.v1_kubeclient(context) + @v1_kubeclient ||= build_v1_kubeclient(context) + end + + def self.v1beta1_kubeclient(context) + @v1beta1_kubeclient ||= build_v1beta1_kubeclient(context) + end + + def self.v1beta1_crd_kubeclient(context) + @v1beta1_kubeclient_crd ||= _build_kubeclient( + api_version: "v1beta1", + context: context, + endpoint_path: "/apis/apiextensions.k8s.io/" + ) + end + end +end diff --git a/lib/kubernetes-deploy/kubeclient_builder.rb b/lib/kubernetes-deploy/kubeclient_builder.rb index b5d3693fd..0aed6366e 100644 --- a/lib/kubernetes-deploy/kubeclient_builder.rb +++ b/lib/kubernetes-deploy/kubeclient_builder.rb @@ -52,7 +52,15 @@ def build_apps_v1beta1_kubeclient(context) ) end - def _build_kubeclient(api_version:, context:, endpoint_path: nil) + def build_apiextensions_v1beta1_kubeclient(context) + _build_kubeclient( + api_version: "v1beta1", + context: context, + endpoint_path: "/apis/apiextensions.k8s.io" + ) + end + + def _build_kubeclient(api_version:, context:, endpoint_path: nil, discover: true) # Find a context defined in kube conf files that matches the input context by name friendly_configs = config_files.map { |f| GoogleFriendlyConfig.read(f) } config = friendly_configs.find { |c| c.contexts.include?(context) } @@ -67,7 +75,7 @@ def _build_kubeclient(api_version:, context:, endpoint_path: nil) ssl_options: kube_context.ssl_options, auth_options: kube_context.auth_options ) - client.discover + client.discover if discover client end diff --git a/lib/kubernetes-deploy/kubernetes_resource.rb b/lib/kubernetes-deploy/kubernetes_resource.rb index 163db86ee..7a663c58d 100644 --- a/lib/kubernetes-deploy/kubernetes_resource.rb +++ b/lib/kubernetes-deploy/kubernetes_resource.rb @@ -7,7 +7,7 @@ module KubernetesDeploy class KubernetesResource attr_reader :name, :namespace, :context - attr_writer :type, :deploy_started_at + attr_writer :deploy_started_at TIMEOUT = 5.minutes LOG_LINE_COUNT = 250 @@ -24,16 +24,50 @@ class KubernetesResource TIMEOUT_OVERRIDE_ANNOTATION = "kubernetes-deploy.shopify.io/timeout-override" - def self.build(namespace:, context:, definition:, logger:) - opts = { namespace: namespace, context: context, definition: definition, logger: logger } - if KubernetesDeploy.const_defined?(definition["kind"]) - klass = KubernetesDeploy.const_get(definition["kind"]) - klass.new(**opts) - else - inst = new(**opts) - inst.type = definition["kind"] - inst - end + def self.inherited(child_class) + child_classes.add(child_class) + end + + def self.child_classes + @@child_classes ||= Set.new + end + + def self.all + child_classes.dup + end + + def self.prunable? + return false unless defined? self::PRUNABLE + self::PRUNABLE + end + + def self.predeploy? + return false unless defined? self::PREDEPLOY + self::PREDEPLOY + end + + def self.group + self::GROUP + end + + def self.version + self::VERSION + end + + def self.kind + self.name.demodulize + end + + def self.qualified_kind + "#{group}/#{version}/#{kind}" + end + + def kind + self.class.kind + end + + def id + "#{kind}/#{name}" end def self.timeout @@ -91,10 +125,6 @@ def validation_failed? @validation_errors.present? end - def id - "#{type}/#{name}" - end - def file_path file.path end @@ -112,7 +142,7 @@ def deploy_started? def deploy_succeeded? if deploy_started? && !@success_assumption_warning_shown - @logger.warn("Don't know how to monitor resources of type #{type}. Assuming #{id} deployed successfully.") + @logger.warn("Don't know how to monitor resources of type #{kind}. Assuming #{id} deployed successfully.") @success_assumption_warning_shown = true end true @@ -126,10 +156,6 @@ def status @status ||= "Unknown" end - def type - @type || self.class.name.demodulize - end - def deploy_timed_out? return false unless deploy_started? !deploy_succeeded? && !deploy_failed? && (Time.now.utc - @deploy_started_at > timeout) @@ -204,7 +230,7 @@ def debug_message # } def fetch_events return {} unless exists? - out, _err, st = kubectl.run("get", "events", "--output=go-template=#{Event.go_template_for(type, name)}") + out, _err, st = kubectl.run("get", "events", "--output=go-template=#{Event.go_template_for(kind, name)}") return {} unless st.success? event_collector = Hash.new { |hash, key| hash[key] = [] } @@ -321,7 +347,7 @@ def file end def create_definition_tempfile - file = Tempfile.new(["#{type}-#{name}", ".yml"]) + file = Tempfile.new(["#{kind}-#{name}", ".yml"]) file.write(YAML.dump(@definition)) file ensure @@ -342,7 +368,7 @@ def statsd_tags else "unknown" end - %W(context:#{context} namespace:#{namespace} resource:#{id} type:#{type} sha:#{ENV['REVISION']} status:#{status}) + %W(context:#{context} namespace:#{namespace} resource:#{id} type:#{kind} sha:#{ENV['REVISION']} status:#{status}) end end end diff --git a/lib/kubernetes-deploy/kubernetes_resource/bucket.rb b/lib/kubernetes-deploy/kubernetes_resource/bucket.rb index eafb57c7a..b918d5eec 100644 --- a/lib/kubernetes-deploy/kubernetes_resource/bucket.rb +++ b/lib/kubernetes-deploy/kubernetes_resource/bucket.rb @@ -2,7 +2,7 @@ module KubernetesDeploy class Bucket < KubernetesResource def sync - _, _err, st = kubectl.run("get", type, @name) + _, _err, st = kubectl.run("get", kind, @name) @found = st.success? end @@ -10,7 +10,7 @@ def deploy_succeeded? return false unless deploy_started? unless @success_assumption_warning_shown - @logger.warn("Don't know how to monitor resources of type #{type}. Assuming #{id} deployed successfully.") + @logger.warn("Don't know how to monitor resources of type #{kind}. Assuming #{id} deployed successfully.") @success_assumption_warning_shown = true end true diff --git a/lib/kubernetes-deploy/kubernetes_resource/bugsnag.rb b/lib/kubernetes-deploy/kubernetes_resource/bugsnag.rb index 1b27e0784..75e1e8834 100644 --- a/lib/kubernetes-deploy/kubernetes_resource/bugsnag.rb +++ b/lib/kubernetes-deploy/kubernetes_resource/bugsnag.rb @@ -2,10 +2,13 @@ module KubernetesDeploy class Bugsnag < KubernetesResource TIMEOUT = 1.minute + PREDEPLOY = true + GROUP = 'stable.shopify.io' + VERSION = 'v1' def sync @secret_found = false - _, _err, st = kubectl.run("get", type, @name) + _, _err, st = kubectl.run("get", kind, @name) @found = st.success? if @found secrets, _err, _st = kubectl.run("get", "secrets", "--output=name") diff --git a/lib/kubernetes-deploy/kubernetes_resource/cloudsql.rb b/lib/kubernetes-deploy/kubernetes_resource/cloudsql.rb index 1ea41db8c..75cea9c16 100644 --- a/lib/kubernetes-deploy/kubernetes_resource/cloudsql.rb +++ b/lib/kubernetes-deploy/kubernetes_resource/cloudsql.rb @@ -2,9 +2,12 @@ module KubernetesDeploy class Cloudsql < KubernetesResource TIMEOUT = 10.minutes + PREDEPLOY = true + GROUP = 'stable.shopify.io' + VERSION = 'v1' def sync - _, _err, st = kubectl.run("get", type, @name) + _, _err, st = kubectl.run("get", kind, @name) @found = st.success? @deployment_exists = cloudsql_proxy_deployment_exists? @service_exists = mysql_service_exists? diff --git a/lib/kubernetes-deploy/kubernetes_resource/config_map.rb b/lib/kubernetes-deploy/kubernetes_resource/config_map.rb index 861373b55..caacf8245 100644 --- a/lib/kubernetes-deploy/kubernetes_resource/config_map.rb +++ b/lib/kubernetes-deploy/kubernetes_resource/config_map.rb @@ -2,9 +2,11 @@ module KubernetesDeploy class ConfigMap < KubernetesResource TIMEOUT = 30.seconds + PREDEPLOY = true + PRUNABLE = true def sync - _, _err, st = kubectl.run("get", type, @name) + _, _err, st = kubectl.run("get", kind, @name) @status = st.success? ? "Available" : "Unknown" @found = st.success? end diff --git a/lib/kubernetes-deploy/kubernetes_resource/cron_job.rb b/lib/kubernetes-deploy/kubernetes_resource/cron_job.rb index bd0889068..7ef0ffc40 100644 --- a/lib/kubernetes-deploy/kubernetes_resource/cron_job.rb +++ b/lib/kubernetes-deploy/kubernetes_resource/cron_job.rb @@ -1,10 +1,11 @@ # frozen_string_literal: true module KubernetesDeploy class CronJob < KubernetesResource + PRUNABLE = true TIMEOUT = 30.seconds def sync - _, _err, st = kubectl.run("get", type, @name) + _, _err, st = kubectl.run("get", kind, @name) @status = st.success? ? "Available" : "Unknown" @found = st.success? end diff --git a/lib/kubernetes-deploy/kubernetes_resource/customresourcedefinition.rb b/lib/kubernetes-deploy/kubernetes_resource/customresourcedefinition.rb new file mode 100644 index 000000000..1eea303d3 --- /dev/null +++ b/lib/kubernetes-deploy/kubernetes_resource/customresourcedefinition.rb @@ -0,0 +1,29 @@ +# frozen_string_literal: true +module KubernetesDeploy + class CustomResourceDefinition < KubernetesResource + TIMEOUT = 10.seconds + PREDEPLOY = true + + def sync + _, _err, st = kubectl.run("get", kind, @name) + @status = st.success? ? "Available" : "Unknown" + @found = st.success? + end + + def deploy_succeeded? + exists? + end + + def deploy_failed? + false + end + + def timeout_message + UNUSUAL_FAILURE_MESSAGE + end + + def exists? + @found + end + end +end diff --git a/lib/kubernetes-deploy/kubernetes_resource/daemon_set.rb b/lib/kubernetes-deploy/kubernetes_resource/daemon_set.rb index ac1557bd8..7c394eca3 100644 --- a/lib/kubernetes-deploy/kubernetes_resource/daemon_set.rb +++ b/lib/kubernetes-deploy/kubernetes_resource/daemon_set.rb @@ -3,10 +3,11 @@ module KubernetesDeploy class DaemonSet < PodSetBase TIMEOUT = 5.minutes + PRUNABLE = true attr_reader :pods def sync - raw_json, _err, st = kubectl.run("get", type, @name, "--output=json") + raw_json, _err, st = kubectl.run("get", kind, @name, "--output=json") @found = st.success? if @found diff --git a/lib/kubernetes-deploy/kubernetes_resource/deployment.rb b/lib/kubernetes-deploy/kubernetes_resource/deployment.rb index 6e67b7780..f1965f2fb 100644 --- a/lib/kubernetes-deploy/kubernetes_resource/deployment.rb +++ b/lib/kubernetes-deploy/kubernetes_resource/deployment.rb @@ -5,9 +5,10 @@ class Deployment < KubernetesResource REQUIRED_ROLLOUT_ANNOTATION = 'kubernetes-deploy.shopify.io/required-rollout' REQUIRED_ROLLOUT_TYPES = %w(maxUnavailable full none).freeze DEFAULT_REQUIRED_ROLLOUT = 'full' + PRUNABLE = true def sync - raw_json, _err, st = kubectl.run("get", type, @name, "--output=json") + raw_json, _err, st = kubectl.run("get", kind, @name, "--output=json") @found = st.success? if @found @@ -80,7 +81,7 @@ def timeout_message reason_msg = if @progress_condition.present? "Timeout reason: #{@progress_condition['reason']}" else - "Timeout reason: hard deadline for #{type}" + "Timeout reason: hard deadline for #{kind}" end return reason_msg unless @latest_rs.present? "#{reason_msg}\nLatest ReplicaSet: #{@latest_rs.name}\n\n#{@latest_rs.timeout_message}" diff --git a/lib/kubernetes-deploy/kubernetes_resource/elasticsearch.rb b/lib/kubernetes-deploy/kubernetes_resource/elasticsearch.rb index 23991c731..1293d972a 100644 --- a/lib/kubernetes-deploy/kubernetes_resource/elasticsearch.rb +++ b/lib/kubernetes-deploy/kubernetes_resource/elasticsearch.rb @@ -1,8 +1,11 @@ # frozen_string_literal: true module KubernetesDeploy class Elasticsearch < KubernetesResource + GROUP = 'stable.shopify.io' + VERSION = 'v1' + def sync - _, _err, st = kubectl.run("get", type, @name) + _, _err, st = kubectl.run("get", kind, @name) @found = st.success? end @@ -10,7 +13,7 @@ def deploy_succeeded? return false unless deploy_started? unless @success_assumption_warning_shown - @logger.warn("Don't know how to monitor resources of type #{type}. Assuming #{id} deployed successfully.") + @logger.warn("Don't know how to monitor resources of type #{kind}. Assuming #{id} deployed successfully.") @success_assumption_warning_shown = true end true diff --git a/lib/kubernetes-deploy/kubernetes_resource/horizontalpodautoscaler.rb b/lib/kubernetes-deploy/kubernetes_resource/horizontalpodautoscaler.rb new file mode 100644 index 000000000..8d006a209 --- /dev/null +++ b/lib/kubernetes-deploy/kubernetes_resource/horizontalpodautoscaler.rb @@ -0,0 +1,29 @@ +# frozen_string_literal: true +module KubernetesDeploy + class HorizontalPodAutoscaler < KubernetesResource + PRUNABLE = true + TIMEOUT = 30.seconds + + def sync + _, _err, st = kubectl.run("get", kind, @name) + @status = st.success? ? "Available" : "Unknown" + @found = st.success? + end + + def deploy_succeeded? + exists? + end + + def deploy_failed? + !exists? + end + + def timeout_message + UNUSUAL_FAILURE_MESSAGE + end + + def exists? + @found + end + end +end diff --git a/lib/kubernetes-deploy/kubernetes_resource/ingress.rb b/lib/kubernetes-deploy/kubernetes_resource/ingress.rb index 790284873..cfb5407ad 100644 --- a/lib/kubernetes-deploy/kubernetes_resource/ingress.rb +++ b/lib/kubernetes-deploy/kubernetes_resource/ingress.rb @@ -2,9 +2,10 @@ module KubernetesDeploy class Ingress < KubernetesResource TIMEOUT = 30.seconds + PRUNABLE = true def sync - _, _err, st = kubectl.run("get", type, @name) + _, _err, st = kubectl.run("get", kind, @name) @status = st.success? ? "Created" : "Unknown" @found = st.success? end diff --git a/lib/kubernetes-deploy/kubernetes_resource/job.rb b/lib/kubernetes-deploy/kubernetes_resource/job.rb new file mode 100644 index 000000000..11cf4a163 --- /dev/null +++ b/lib/kubernetes-deploy/kubernetes_resource/job.rb @@ -0,0 +1,29 @@ +# frozen_string_literal: true +module KubernetesDeploy + class Job < KubernetesResource + PRUNABLE = true + TIMEOUT = 30.seconds + + def sync + _, _err, st = kubectl.run("get", kind, @name) + @status = st.success? ? "Available" : "Unknown" + @found = st.success? + end + + def deploy_succeeded? + exists? + end + + def deploy_failed? + !exists? + end + + def timeout_message + UNUSUAL_FAILURE_MESSAGE + end + + def exists? + @found + end + end +end diff --git a/lib/kubernetes-deploy/kubernetes_resource/memcached.rb b/lib/kubernetes-deploy/kubernetes_resource/memcached.rb index 4bda9fa4f..fe52d73bc 100644 --- a/lib/kubernetes-deploy/kubernetes_resource/memcached.rb +++ b/lib/kubernetes-deploy/kubernetes_resource/memcached.rb @@ -2,10 +2,13 @@ module KubernetesDeploy class Memcached < KubernetesResource TIMEOUT = 5.minutes + PREDEPLOY = true CONFIGMAP_NAME = "memcached-url" + GROUP = 'stable.shopify.io' + VERSION = 'v1' def sync - _, _err, st = kubectl.run("get", type, @name) + _, _err, st = kubectl.run("get", kind, @name) @found = st.success? @deployment_exists = memcached_deployment_exists? @service_exists = memcached_service_exists? diff --git a/lib/kubernetes-deploy/kubernetes_resource/persistent_volume_claim.rb b/lib/kubernetes-deploy/kubernetes_resource/persistent_volume_claim.rb index 22a82643e..a551ca658 100644 --- a/lib/kubernetes-deploy/kubernetes_resource/persistent_volume_claim.rb +++ b/lib/kubernetes-deploy/kubernetes_resource/persistent_volume_claim.rb @@ -2,9 +2,10 @@ module KubernetesDeploy class PersistentVolumeClaim < KubernetesResource TIMEOUT = 5.minutes + PREDEPLOY = true def sync - out, _err, st = kubectl.run("get", type, @name, "--output=jsonpath={.status.phase}") + out, _err, st = kubectl.run("get", kind, @name, "--output=jsonpath={.status.phase}") @found = st.success? @status = out if @found end diff --git a/lib/kubernetes-deploy/kubernetes_resource/pod.rb b/lib/kubernetes-deploy/kubernetes_resource/pod.rb index f022bb3ff..2a71422e7 100644 --- a/lib/kubernetes-deploy/kubernetes_resource/pod.rb +++ b/lib/kubernetes-deploy/kubernetes_resource/pod.rb @@ -2,7 +2,8 @@ module KubernetesDeploy class Pod < KubernetesResource TIMEOUT = 10.minutes - + PREDEPLOY = true + PRUNABLE = true FAILED_PHASE_NAME = "Failed" def initialize(namespace:, context:, definition:, logger:, parent: nil, deploy_started_at: nil) @@ -19,7 +20,7 @@ def initialize(namespace:, context:, definition:, logger:, parent: nil, deploy_s def sync(pod_data = nil) if pod_data.blank? - raw_json, _err, st = kubectl.run("get", type, @name, "-a", "--output=json") + raw_json, _err, st = kubectl.run("get", kind, @name, "-a", "--output=json") pod_data = JSON.parse(raw_json) if st.success? raise_predates_deploy_error if pod_data.present? && unmanaged? && !deploy_started? end diff --git a/lib/kubernetes-deploy/kubernetes_resource/pod_disruption_budget.rb b/lib/kubernetes-deploy/kubernetes_resource/pod_disruption_budget.rb index 6753b9976..a416045cf 100644 --- a/lib/kubernetes-deploy/kubernetes_resource/pod_disruption_budget.rb +++ b/lib/kubernetes-deploy/kubernetes_resource/pod_disruption_budget.rb @@ -4,7 +4,7 @@ class PodDisruptionBudget < KubernetesResource TIMEOUT = 10.seconds def sync - _, _err, st = kubectl.run("get", type, @name) + _, _err, st = kubectl.run("get", kind, @name) @found = st.success? @status = @found ? "Available" : "Unknown" end diff --git a/lib/kubernetes-deploy/kubernetes_resource/pod_set_base.rb b/lib/kubernetes-deploy/kubernetes_resource/pod_set_base.rb index 6235871c4..21f4bc042 100644 --- a/lib/kubernetes-deploy/kubernetes_resource/pod_set_base.rb +++ b/lib/kubernetes-deploy/kubernetes_resource/pod_set_base.rb @@ -59,7 +59,7 @@ def find_pods(pod_controller_data) context: context, definition: pod_data, logger: @logger, - parent: "#{name.capitalize} #{type}", + parent: "#{name.capitalize} #{kind}", deploy_started_at: @deploy_started_at ) pod.sync(pod_data) diff --git a/lib/kubernetes-deploy/kubernetes_resource/pod_template.rb b/lib/kubernetes-deploy/kubernetes_resource/pod_template.rb index 673ab8711..7ddc60fc7 100644 --- a/lib/kubernetes-deploy/kubernetes_resource/pod_template.rb +++ b/lib/kubernetes-deploy/kubernetes_resource/pod_template.rb @@ -2,7 +2,7 @@ module KubernetesDeploy class PodTemplate < KubernetesResource def sync - _, _err, st = kubectl.run("get", type, @name) + _, _err, st = kubectl.run("get", kind, @name) @status = st.success? ? "Available" : "Unknown" @found = st.success? end diff --git a/lib/kubernetes-deploy/kubernetes_resource/redis.rb b/lib/kubernetes-deploy/kubernetes_resource/redis.rb index f07787003..cd7ff11ff 100644 --- a/lib/kubernetes-deploy/kubernetes_resource/redis.rb +++ b/lib/kubernetes-deploy/kubernetes_resource/redis.rb @@ -2,10 +2,13 @@ module KubernetesDeploy class Redis < KubernetesResource TIMEOUT = 5.minutes + PREDEPLOY = true UUID_ANNOTATION = "redis.stable.shopify.io/owner_uid" + GROUP = 'stable.shopify.io' + VERSION = 'v1' def sync - _, _err, st = kubectl.run("get", type, @name) + _, _err, st = kubectl.run("get", kind, @name) @found = st.success? @deployment_exists = redis_deployment_exists? @service_exists = redis_service_exists? diff --git a/lib/kubernetes-deploy/kubernetes_resource/replica_set.rb b/lib/kubernetes-deploy/kubernetes_resource/replica_set.rb index e4fbf6f0f..9e8dc2042 100644 --- a/lib/kubernetes-deploy/kubernetes_resource/replica_set.rb +++ b/lib/kubernetes-deploy/kubernetes_resource/replica_set.rb @@ -18,7 +18,7 @@ def initialize(namespace:, context:, definition:, logger:, parent: nil, deploy_s def sync(rs_data = nil) if rs_data.blank? - raw_json, _err, st = kubectl.run("get", type, @name, "--output=json") + raw_json, _err, st = kubectl.run("get", kind, @name, "--output=json") rs_data = JSON.parse(raw_json) if st.success? end diff --git a/lib/kubernetes-deploy/kubernetes_resource/resource_quota.rb b/lib/kubernetes-deploy/kubernetes_resource/resource_quota.rb index b98fbb310..2187d1e50 100644 --- a/lib/kubernetes-deploy/kubernetes_resource/resource_quota.rb +++ b/lib/kubernetes-deploy/kubernetes_resource/resource_quota.rb @@ -2,9 +2,10 @@ module KubernetesDeploy class ResourceQuota < KubernetesResource TIMEOUT = 30.seconds + PREDEPLOY = true def sync - raw_json, _err, st = kubectl.run("get", type, @name, "--output=json") + raw_json, _err, st = kubectl.run("get", kind, @name, "--output=json") @status = st.success? ? "Available" : "Unknown" @found = st.success? @rollout_data = if @found diff --git a/lib/kubernetes-deploy/kubernetes_resource/service.rb b/lib/kubernetes-deploy/kubernetes_resource/service.rb index c1317accf..4d0c63873 100644 --- a/lib/kubernetes-deploy/kubernetes_resource/service.rb +++ b/lib/kubernetes-deploy/kubernetes_resource/service.rb @@ -1,10 +1,11 @@ # frozen_string_literal: true module KubernetesDeploy class Service < KubernetesResource + PRUNABLE = true TIMEOUT = 7.minutes def sync - _, _err, st = kubectl.run("get", type, @name) + _, _err, st = kubectl.run("get", kind, @name) @found = st.success? @related_deployment_replicas = fetch_related_replica_count @num_pods_selected = fetch_related_pod_count diff --git a/lib/kubernetes-deploy/kubernetes_resource/service_account.rb b/lib/kubernetes-deploy/kubernetes_resource/service_account.rb index 9f2ff6ab3..55cc6198b 100644 --- a/lib/kubernetes-deploy/kubernetes_resource/service_account.rb +++ b/lib/kubernetes-deploy/kubernetes_resource/service_account.rb @@ -2,9 +2,10 @@ module KubernetesDeploy class ServiceAccount < KubernetesResource TIMEOUT = 30.seconds + PREDEPLOY = true def sync - _, _err, st = kubectl.run("get", type, @name, "--output=json") + _, _err, st = kubectl.run("get", kind, @name, "--output=json") @status = st.success? ? "Created" : "Unknown" @found = st.success? end diff --git a/lib/kubernetes-deploy/kubernetes_resource/stateful_set.rb b/lib/kubernetes-deploy/kubernetes_resource/stateful_set.rb index 211927350..0bd25c868 100644 --- a/lib/kubernetes-deploy/kubernetes_resource/stateful_set.rb +++ b/lib/kubernetes-deploy/kubernetes_resource/stateful_set.rb @@ -3,11 +3,12 @@ module KubernetesDeploy class StatefulSet < PodSetBase TIMEOUT = 10.minutes + PRUNABLE = true ONDELETE = 'OnDelete' attr_reader :pods def sync - raw_json, _err, st = kubectl.run("get", type, @name, "--output=json") + raw_json, _err, st = kubectl.run("get", kind, @name, "--output=json") @found = st.success? if @found diff --git a/lib/kubernetes-deploy/kubernetes_resource/statefulservice.rb b/lib/kubernetes-deploy/kubernetes_resource/statefulservice.rb index 69d945ae2..00c8c475e 100644 --- a/lib/kubernetes-deploy/kubernetes_resource/statefulservice.rb +++ b/lib/kubernetes-deploy/kubernetes_resource/statefulservice.rb @@ -1,8 +1,11 @@ # frozen_string_literal: true module KubernetesDeploy class Statefulservice < KubernetesResource + GROUP = 'stable.shopify.io' + VERSION = 'v1' + def sync - _, _err, st = kubectl.run("get", type, @name) + _, _err, st = kubectl.run("get", kind, @name) @found = st.success? end @@ -10,7 +13,7 @@ def deploy_succeeded? return false unless deploy_started? unless @success_assumption_warning_shown - @logger.warn("Don't know how to monitor resources of type #{type}. Assuming #{id} deployed successfully.") + @logger.warn("Don't know how to monitor resources of type #{kind}. Assuming #{id} deployed successfully.") @success_assumption_warning_shown = true end true diff --git a/lib/kubernetes-deploy/kubernetes_resource/topic.rb b/lib/kubernetes-deploy/kubernetes_resource/topic.rb index ff7facb16..31250e66f 100644 --- a/lib/kubernetes-deploy/kubernetes_resource/topic.rb +++ b/lib/kubernetes-deploy/kubernetes_resource/topic.rb @@ -2,7 +2,7 @@ module KubernetesDeploy class Topic < KubernetesResource def sync - _, _err, st = kubectl.run("get", type, @name) + _, _err, st = kubectl.run("get", kind, @name) @found = st.success? end @@ -10,7 +10,7 @@ def deploy_succeeded? return false unless deploy_started? unless @success_assumption_warning_shown - @logger.warn("Don't know how to monitor resources of type #{type}. Assuming #{id} deployed successfully.") + @logger.warn("Don't know how to monitor resources of type #{kind}. Assuming #{id} deployed successfully.") @success_assumption_warning_shown = true end true diff --git a/test/fixtures/resource-discovery/definitions/crd.yml b/test/fixtures/resource-discovery/definitions/crd.yml new file mode 100644 index 000000000..effb86f03 --- /dev/null +++ b/test/fixtures/resource-discovery/definitions/crd.yml @@ -0,0 +1,14 @@ +--- +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: widgets.api.foobar.com + annotations: + kubernetes-deploy.shopify.io/metadata: '{"prunable":"true","predeploy":"true","timeout":"PT9M","status-field":"status","status-success":"ok"}' +spec: + group: api.foobar.com + version: v1 + names: + kind: Widget + plural: widgets + scope: Namespaced diff --git a/test/fixtures/resource-discovery/definitions/crd_non_prunable_no_predeploy.yml b/test/fixtures/resource-discovery/definitions/crd_non_prunable_no_predeploy.yml new file mode 100644 index 000000000..b24dc7d30 --- /dev/null +++ b/test/fixtures/resource-discovery/definitions/crd_non_prunable_no_predeploy.yml @@ -0,0 +1,14 @@ +--- +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: widgets.api.foobar.com + annotations: + kubernetes-deploy.shopify.io/metadata: '{"status-field":"status","status-success":"ok"}' +spec: + group: api.foobar.com + version: v1 + names: + kind: Widget + plural: widgets + scope: Namespaced diff --git a/test/fixtures/resource-discovery/instances/crd.yml b/test/fixtures/resource-discovery/instances/crd.yml new file mode 100644 index 000000000..e08049694 --- /dev/null +++ b/test/fixtures/resource-discovery/instances/crd.yml @@ -0,0 +1,6 @@ +--- +apiVersion: "api.foobar.com/v1" +kind: Widget +metadata: + name: my-first-widget +status: "ok" diff --git a/test/helpers/kubeclient_helper.rb b/test/helpers/kubeclient_helper.rb index 3e5b3a040..5cbe06755 100644 --- a/test/helpers/kubeclient_helper.rb +++ b/test/helpers/kubeclient_helper.rb @@ -18,6 +18,10 @@ def policy_v1beta1_kubeclient @policy_v1beta1_kubeclient ||= build_policy_v1beta1_kubeclient(MINIKUBE_CONTEXT) end + def apiextensions_v1beta1_kubeclient + @apiextensions_v1beta1_kubeclient ||= build_apiextensions_v1beta1_kubeclient(MINIKUBE_CONTEXT) + end + def apps_v1beta1_kubeclient @apps_v1beta1_kubeclient ||= build_apps_v1beta1_kubeclient(MINIKUBE_CONTEXT) end diff --git a/test/integration/resource_discovery_test.rb b/test/integration/resource_discovery_test.rb new file mode 100644 index 000000000..6c815337a --- /dev/null +++ b/test/integration/resource_discovery_test.rb @@ -0,0 +1,54 @@ +# frozen_string_literal: true +require 'test_helper' + +class ResourceDiscoveryTest < KubernetesDeploy::IntegrationTest + def cleanup(*resources) + crd_list = apiextensions_v1beta1_kubeclient.get_custom_resource_definitions + crd_list.each do |res| + apiextensions_v1beta1_kubeclient.delete_custom_resource_definition res.metadata.name + end + end + + # Test resource generation + # TODO + + # Test FatalDeploymentError, "Invalid metadata content: + + def test_non_prunable_crd_no_predeploy + skip if KUBE_SERVER_VERSION < Gem::Version.new('1.7.0') + begin + assert_deploy_success(deploy_fixtures("resource-discovery/definitions", + subset: ["crd_non_prunable_no_predeploy.yml"])) + assert_deploy_success(deploy_fixtures("resource-discovery/instances", subset: ["crd.yml"])) + # Deploy any other non-priority (predeployable) resource to trigger pruning + assert_deploy_success(deploy_fixtures("hello-cloud", subset: ["daemon_set.yml",])) + + refute_logs_match("The following resources were pruned: widget \"my-first-widget\"") + refute_logs_match("Don't know how to monitor resources of type Widget. " \ + "Assuming Widget/my-first-widget deployed successfully") + # Should predeploy the CR definition, but *not* the instance. + assert_logs_match(/Predeploying priority resources\-+\s+\[INFO\]\[(\d|\-|\s|\:)+\]\s+Deploying CustomResourceDefinition\/widgets\.api\.foobar\.com/) + refute_logs_match(/Predeploying priority resources\-+\s+\[INFO\]\[(\d|\-|\s|\:)+\]\s+Deploying Widget\/my\-first\-widget/) + ensure + cleanup + end + end + + def test_prunable_crd_with_predeploy + skip if KUBE_SERVER_VERSION < Gem::Version.new('1.7.0') + begin + assert_deploy_success(deploy_fixtures("resource-discovery/definitions", subset: ["crd.yml"])) + assert_deploy_success(deploy_fixtures("resource-discovery/instances", subset: ["crd.yml"])) + # Deploy any other resource to trigger pruning + assert_deploy_success(deploy_fixtures("hello-cloud", subset: ["configmap-data.yml",])) + # Should predeploy the CR definition, *and* the instance. + assert_logs_match(/Predeploying priority resources\-+\s+\[INFO\]\[(\d|\-|\s|\:)+\]\s+Deploying CustomResourceDefinition\/widgets\.api\.foobar\.com/) + assert_logs_match(/Predeploying priority resources\-+\s+\[INFO\]\[(\d|\-|\s|\:)+\]\s+Deploying Widget\/my\-first\-widget/) + assert_logs_match("The following resources were pruned: widget \"my-first-widget\"") + refute_logs_match("Don't know how to monitor resources of type Widget. " \ + "Assuming Widget/my-first-widget deployed successfully") + ensure + cleanup + end + end +end