From 7e19140cab9e433e660b836593e8e7a5a748114f Mon Sep 17 00:00:00 2001 From: Danny Turner Date: Fri, 3 Aug 2018 12:02:50 -0700 Subject: [PATCH] Pruneable CRs (#312) * Add the ability for crd pruning Update readme and change-log --- CHANGELOG.md | 11 ++++- README.md | 4 ++ lib/kubernetes-deploy/deploy_task.rb | 9 +++- .../custom_resource_definition.rb | 15 ++++++ lib/kubernetes-deploy/resource_discovery.rb | 19 ++++++++ test/fixtures/crd/configmap-data.yml | 10 ++++ test/fixtures/crd/configmap2.yml | 10 ++++ test/fixtures/crd/{crd.yml.erb => mail.yml} | 0 test/fixtures/crd/mail_cr.yml | 4 ++ test/fixtures/crd/widgets.yml | 13 +++++ test/fixtures/crd/widgets_cr.yml | 5 ++ test/integration-serial/run_serial_test.rb | 47 ++++++++++++------- 12 files changed, 126 insertions(+), 21 deletions(-) create mode 100644 lib/kubernetes-deploy/resource_discovery.rb create mode 100644 test/fixtures/crd/configmap-data.yml create mode 100644 test/fixtures/crd/configmap2.yml rename test/fixtures/crd/{crd.yml.erb => mail.yml} (100%) create mode 100644 test/fixtures/crd/mail_cr.yml create mode 100644 test/fixtures/crd/widgets.yml create mode 100644 test/fixtures/crd/widgets_cr.yml diff --git a/CHANGELOG.md b/CHANGELOG.md index ffe63484a..6efb437f3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,15 @@ ### Master +*Enhancements* +- Add Job resource class ([#295](https://github.com/Shopify/kubernetes-deploy/pull/296)) +- Add CustomResourceDefinition resource class ([#305](https://github.com/Shopify/kubernetes-deploy/pull/306)) +- Officially support Kubernetes 1.10 ([#308](https://github.com/Shopify/kubernetes-deploy/pull/308)) +- SyncMediator will only batch fetch resources when there is a sufficiently large enough set of resources +being tracked ([#316](https://github.com/Shopify/kubernetes-deploy/pull/316)) +- Allow CRs to be pruned based on `kubernetes-deploy.shopify.io/prunable` annotation on the custom resource definitions ([312](https://github.com/Shopify/kubernetes-deploy/pull/312)) + ### 0.20.4 +*Enhancements* - Don't consider pod preempting a failure ([#317](https://github.com/shopify/kubernetes-deploy/pull/317)) ### 0.20.3 @@ -8,7 +17,6 @@ - Evictions are recoverable so prevent them from triggering fast failure detection ([#293](https://github.com/Shopify/kubernetes-deploy/pull/293)). - Use YAML.safe_load over YAML.load_file ([#295](https://github.com/Shopify/kubernetes-deploy/pull/295)). - *Bug Fixes* - Default rollout strategy is compatible required-rollout annotation ([#289](https://github.com/Shopify/kubernetes-deploy/pull/289)). @@ -16,7 +24,6 @@ *Enhancements* - Emit data dog events when deploys succeed, time out or fail ([#292](https://github.com/Shopify/kubernetes-deploy/pull/292)). ### 0.20.1 -*Features* *Bug Fixes* - Display a nice error instead of crashing when a YAML document is missing 'Kind' diff --git a/README.md b/README.md index 1281d4f4c..41272ac27 100644 --- a/README.md +++ b/README.md @@ -241,6 +241,10 @@ before the deployment is considered successful. that use the `RollingUpdate` strategy. - Percent (e.g. 90%): The deploy is successful when the number of new pods that are ready is equal to `spec.replicas` * Percent. +- `kubernetes-deploy.shopify.io/prunable`: Allows a Custom Resource to be pruned during deployment. + - _Compatibility_: Custom Resource Definition + - `true`: The custom resource will be pruned if the resource is not in the deploy directory. + - All other values: The custom resource will not be pruned. ### Running tasks at the beginning of a deploy diff --git a/lib/kubernetes-deploy/deploy_task.rb b/lib/kubernetes-deploy/deploy_task.rb index 318b36f3c..61155e604 100644 --- a/lib/kubernetes-deploy/deploy_task.rb +++ b/lib/kubernetes-deploy/deploy_task.rb @@ -37,6 +37,7 @@ require 'kubernetes-deploy/kubeclient_builder' require 'kubernetes-deploy/ejson_secret_provisioner' require 'kubernetes-deploy/renderer' +require 'kubernetes-deploy/resource_discovery' module KubernetesDeploy class DeployTask @@ -52,6 +53,7 @@ class DeployTask ServiceAccount Pod ) + PROTECTED_NAMESPACES = %w( default kube-system @@ -65,6 +67,7 @@ class DeployTask # core/v1/ReplicationController -- superseded by deployments/replicasets # extensions/v1beta1/ReplicaSet -- managed by deployments # core/v1/Secret -- should not committed / managed by shipit + def prune_whitelist wl = %w( core/v1/ConfigMap @@ -82,7 +85,7 @@ def prune_whitelist if server_version >= Gem::Version.new('1.8.0') wl << "batch/v1beta1/CronJob" end - wl + wl + cluster_resource_discoverer.crds(@sync_mediator).select(&:prunable?).map(&:group_version_kind) end def server_version @@ -200,6 +203,10 @@ def run!(verify_result: true, allow_protected_ns: false, prune: true) private + def cluster_resource_discoverer + ResourceDiscovery.new(namespace: @namespace, context: @context, logger: @logger, namespace_tags: @namespace_tags) + end + def deploy_has_priority_resources?(resources) resources.any? { |r| PREDEPLOY_SEQUENCE.include?(r.type) } end diff --git a/lib/kubernetes-deploy/kubernetes_resource/custom_resource_definition.rb b/lib/kubernetes-deploy/kubernetes_resource/custom_resource_definition.rb index 9d7767d9a..017712602 100644 --- a/lib/kubernetes-deploy/kubernetes_resource/custom_resource_definition.rb +++ b/lib/kubernetes-deploy/kubernetes_resource/custom_resource_definition.rb @@ -26,6 +26,21 @@ def status end end + def group_version_kind + group = @definition.dig("spec", "group") + version = @definition.dig("spec", "version") + "#{group}/#{version}/#{kind}" + end + + def kind + @definition.dig("spec", "names", "kind") + end + + def prunable? + prunable = @definition.dig("metadata", "annotations", "kubernetes-deploy.shopify.io/prunable") + prunable == "true" + end + private def names_accepted_condition diff --git a/lib/kubernetes-deploy/resource_discovery.rb b/lib/kubernetes-deploy/resource_discovery.rb new file mode 100644 index 000000000..7d6492cd4 --- /dev/null +++ b/lib/kubernetes-deploy/resource_discovery.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +module KubernetesDeploy + class ResourceDiscovery + def initialize(namespace:, context:, logger:, namespace_tags:) + @namespace = namespace + @context = context + @logger = logger + @namespace_tags = namespace_tags + end + + def crds(sync_mediator) + sync_mediator.get_all(CustomResourceDefinition.kind).map do |r_def| + CustomResourceDefinition.new(namespace: @namespace, context: @context, logger: @logger, + definition: r_def, statsd_tags: @namespace_tags) + end + end + end +end diff --git a/test/fixtures/crd/configmap-data.yml b/test/fixtures/crd/configmap-data.yml new file mode 100644 index 000000000..c7f84cb47 --- /dev/null +++ b/test/fixtures/crd/configmap-data.yml @@ -0,0 +1,10 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: hello-cloud-configmap-data + labels: + name: hello-cloud-configmap-data + app: hello-cloud +data: + datapoint1: value1 + datapoint2: value2 diff --git a/test/fixtures/crd/configmap2.yml b/test/fixtures/crd/configmap2.yml new file mode 100644 index 000000000..a849d4314 --- /dev/null +++ b/test/fixtures/crd/configmap2.yml @@ -0,0 +1,10 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: crd-configmap-data + labels: + name: crd-configmap-data + app: hello-cloud +data: + datapoint1: value1 + datapoint2: value2 diff --git a/test/fixtures/crd/crd.yml.erb b/test/fixtures/crd/mail.yml similarity index 100% rename from test/fixtures/crd/crd.yml.erb rename to test/fixtures/crd/mail.yml diff --git a/test/fixtures/crd/mail_cr.yml b/test/fixtures/crd/mail_cr.yml new file mode 100644 index 000000000..5582f16ad --- /dev/null +++ b/test/fixtures/crd/mail_cr.yml @@ -0,0 +1,4 @@ +apiVersion: "stable.example.io/v1" +kind: Mail +metadata: + name: my-first-mail diff --git a/test/fixtures/crd/widgets.yml b/test/fixtures/crd/widgets.yml new file mode 100644 index 000000000..15744b56c --- /dev/null +++ b/test/fixtures/crd/widgets.yml @@ -0,0 +1,13 @@ +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: widgets.stable.example.io + annotations: + kubernetes-deploy.shopify.io/prunable: "true" +spec: + group: stable.example.io + version: v1 + names: + kind: Widget + plural: widgets + scope: Namespaced diff --git a/test/fixtures/crd/widgets_cr.yml b/test/fixtures/crd/widgets_cr.yml new file mode 100644 index 000000000..19544289d --- /dev/null +++ b/test/fixtures/crd/widgets_cr.yml @@ -0,0 +1,5 @@ +--- +apiVersion: "stable.example.io/v1" +kind: Widget +metadata: + name: my-first-widget diff --git a/test/integration-serial/run_serial_test.rb b/test/integration-serial/run_serial_test.rb index 16213a685..14bac51f9 100644 --- a/test/integration-serial/run_serial_test.rb +++ b/test/integration-serial/run_serial_test.rb @@ -119,30 +119,16 @@ def test_multiple_configuration_files ENV['KUBECONFIG'] = old_config end - def test_crd_can_be_successful - assert_deploy_success(deploy_fixtures("crd")) - assert_logs_match_all([ - "Phase 1: Initializing deploy", - "Detected non-namespaced resource which will never be pruned:", - " - CustomResourceDefinition/mail.stable.example.io", - "Phase 2: Checking initial resource statuses", - "Deploying CustomResourceDefinition/mail.stable.example.io (timeout: 120s)", - %r{CustomResourceDefinition/mail.stable.example.io\s+Names accepted} - ]) - ensure - apiextensions_v1beta1_kubeclient.delete_custom_resource_definition("mail.stable.example.io") - end - def test_crd_can_fail - result = deploy_fixtures("crd") do |f| - crd = f.dig("crd.yml.erb", "CustomResourceDefinition").first + result = deploy_fixtures("crd", subset: %w(mail.yml)) do |f| + crd = f.dig("mail.yml", "CustomResourceDefinition").first names = crd.dig("spec", "names") names["listKind"] = 'Conflict' end assert_deploy_success(result) - result = deploy_fixtures("crd") do |f| - crd = f.dig("crd.yml.erb", "CustomResourceDefinition").first + result = deploy_fixtures("crd", subset: %w(mail.yml)) do |f| + crd = f.dig("mail.yml", "CustomResourceDefinition").first names = crd.dig("spec", "names") names["listKind"] = "Conflict" names["plural"] = "others" @@ -158,4 +144,29 @@ def test_crd_can_fail apiextensions_v1beta1_kubeclient.delete_custom_resource_definition("mail.stable.example.io") apiextensions_v1beta1_kubeclient.delete_custom_resource_definition("others.stable.example.io") end + + def test_crd_pruning + assert_deploy_success(deploy_fixtures("crd", subset: %w(mail.yml widgets.yml))) + assert_logs_match_all([ + "Phase 1: Initializing deploy", + "Detected non-namespaced resources which will never be pruned:", + " - CustomResourceDefinition/mail.stable.example.io", + "Phase 3: Deploying all resources", + "CustomResourceDefinition/mail.stable.example.io (timeout: 120s)", + %r{CustomResourceDefinition/mail.stable.example.io\s+Names accepted} + ]) + assert_deploy_success(deploy_fixtures("crd", subset: %w(mail_cr.yml widgets_cr.yml configmap-data.yml))) + # Deploy any other non-priority (predeployable) resource to trigger pruning + assert_deploy_success(deploy_fixtures("crd", subset: %w(configmap-data.yml configmap2.yml))) + + assert_predicate build_kubectl.run("get", "mail.stable.example.io", "my-first-mail").last, :success? + refute_logs_match(/The following resources were pruned: mail(.stable.example.io)? "my-first-mail"/) + assert_logs_match_all([ + /The following resources were pruned: widget(.stable.example.io)? "my-first-widget"/, + "Pruned 1 resource and successfully deployed 2 resource" + ]) + ensure + apiextensions_v1beta1_kubeclient.delete_custom_resource_definition("mail.stable.example.io") + apiextensions_v1beta1_kubeclient.delete_custom_resource_definition("widgets.stable.example.io") + end end