From f6cf7913e0aaef5435ee0f0a1c99d8a1641f9b44 Mon Sep 17 00:00:00 2001 From: Katrina Verey Date: Wed, 24 Oct 2018 15:50:43 -0400 Subject: [PATCH 1/2] Support test clusters other than minikube --- .gitignore | 1 + README.md | 15 +++++++++---- test/fixtures/kube-config/invalid_config.yml | 19 ++++++++++------ test/helpers/fixture_deploy_helper.rb | 4 ++-- test/helpers/kubeclient_helper.rb | 22 +++++++++++-------- test/helpers/task_runner_test_helper.rb | 2 +- test/integration-serial/serial_deploy_test.rb | 2 +- test/integration/kubernetes_deploy_test.rb | 18 ++++++++++++--- test/integration/restart_task_test.rb | 6 ++--- test/integration/runner_task_test.rb | 2 +- test/test_helper.rb | 12 +++++----- .../ejson_secret_provisioner_test.rb | 2 +- .../kubeclient_builder_test.rb | 2 +- .../kubernetes-deploy/runner_task_test.rb | 4 ++-- test/unit/sync_mediator_test.rb | 2 +- 15 files changed, 71 insertions(+), 42 deletions(-) diff --git a/.gitignore b/.gitignore index a34ad4900..6a3ecc8ba 100644 --- a/.gitignore +++ b/.gitignore @@ -17,3 +17,4 @@ coverage/* dev/flamegraph.svg dev/profile dev/flamegraph.pl +.local-context diff --git a/README.md b/README.md index 97a7af60e..8a49f7b74 100644 --- a/README.md +++ b/README.md @@ -385,16 +385,23 @@ To install this gem onto your local machine, run `bundle exec rake install`. ## Running the test suite locally -1. Start [minikube](https://kubernetes.io/docs/getting-started-guides/minikube/#installation) (`minikube start [options]`) -2. Make sure you have a context named "minikube" in your kubeconfig. Minikube adds this context for you when you run `minikube start`; please do not rename it. You can check for it using `kubectl config get-contexts`. -3. Run `bundle exec rake test` +Using minikube: +1. Start [minikube](https://kubernetes.io/docs/getting-started-guides/minikube/#installation) (`minikube start [options]`). +2. Make sure you have a context named "minikube" in your kubeconfig. Minikube adds this context for you when you run `minikube start`. You can check for it using `kubectl config get-contexts`. +3. Run `bundle exec rake test` (or `dev test` if you work for Shopify). +Using another local cluster: -To see the full-color output of a specific integration test, you can use `PRINT_LOGS=1 bundle exec ruby -I test test/integration/kubernetes_deploy_test.rb -n/test_name/`. +1. Start your cluster. +2. Put the name of the context you want to use in a file named `.local-context` in the root of this project. For example: `echo "dind" > .local-context`. +3. Run `bundle exec rake test` (or `dev test` if you work for Shopify). To make StatsD log what it would have emitted, run a test with `STATSD_DEV=1`. +To see the full-color output of a specific integration test, you can use `PRINT_LOGS=1`. For example: `PRINT_LOGS=1 bundle exec ruby -I test test/integration/kubernetes_deploy_test.rb -n/test_name/`. + + ![test-output](screenshots/test-output.png) diff --git a/test/fixtures/kube-config/invalid_config.yml b/test/fixtures/kube-config/invalid_config.yml index 53d774747..8edf89b83 100644 --- a/test/fixtures/kube-config/invalid_config.yml +++ b/test/fixtures/kube-config/invalid_config.yml @@ -3,18 +3,23 @@ clusters: - cluster: insecure-skip-tls-verify: true server: https://192.168.5.1 - name: minikube + name: invalid contexts: - context: - cluster: minikube - namespace: default - user: minikube + cluster: invalid + user: test-user name: minikube -current-context: "" +- context: + cluster: invalid + user: test-user + name: dind +- context: + cluster: invalid + user: test-user + name: docker-for-mac kind: Config -preferences: {} users: -- name: minikube +- name: test-user user: username: username password: password diff --git a/test/helpers/fixture_deploy_helper.rb b/test/helpers/fixture_deploy_helper.rb index f59f507ad..2866313c4 100644 --- a/test/helpers/fixture_deploy_helper.rb +++ b/test/helpers/fixture_deploy_helper.rb @@ -68,7 +68,7 @@ def deploy_dir_without_profiling(dir, wait: true, allow_protected_ns: false, pru deploy = KubernetesDeploy::DeployTask.new( namespace: @namespace, current_sha: current_sha, - context: KubeclientHelper::MINIKUBE_CONTEXT, + context: KubeclientHelper::TEST_CONTEXT, template_dir: dir, logger: logger, kubectl_instance: kubectl_instance, @@ -129,7 +129,7 @@ def write_fixtures_to_dir(fixtures, target_dir) end def build_kubectl(log_failure_by_default: true, timeout: '5s') - KubernetesDeploy::Kubectl.new(namespace: @namespace, context: KubeclientHelper::MINIKUBE_CONTEXT, logger: logger, + KubernetesDeploy::Kubectl.new(namespace: @namespace, context: KubeclientHelper::TEST_CONTEXT, logger: logger, log_failure_by_default: log_failure_by_default, default_timeout: timeout) end end diff --git a/test/helpers/kubeclient_helper.rb b/test/helpers/kubeclient_helper.rb index 3eccc72cf..acd9fb33b 100644 --- a/test/helpers/kubeclient_helper.rb +++ b/test/helpers/kubeclient_helper.rb @@ -2,39 +2,43 @@ require 'kubernetes-deploy/kubeclient_builder' module KubeclientHelper - MINIKUBE_CONTEXT = "minikube" + LOCAL_CONTEXT_OVERRIDE_PATH = File.expand_path("../../.local-context", __dir__) + if File.exist?(LOCAL_CONTEXT_OVERRIDE_PATH) + TEST_CONTEXT = File.read(LOCAL_CONTEXT_OVERRIDE_PATH).split.first + end + TEST_CONTEXT ||= "minikube" include KubernetesDeploy::KubeclientBuilder def kubeclient - @kubeclient ||= build_v1_kubeclient(MINIKUBE_CONTEXT) + @kubeclient ||= build_v1_kubeclient(TEST_CONTEXT) end def v1beta1_kubeclient - @v1beta1_kubeclient ||= build_v1beta1_kubeclient(MINIKUBE_CONTEXT) + @v1beta1_kubeclient ||= build_v1beta1_kubeclient(TEST_CONTEXT) end def policy_v1beta1_kubeclient - @policy_v1beta1_kubeclient ||= build_policy_v1beta1_kubeclient(MINIKUBE_CONTEXT) + @policy_v1beta1_kubeclient ||= build_policy_v1beta1_kubeclient(TEST_CONTEXT) end def apps_v1beta1_kubeclient - @apps_v1beta1_kubeclient ||= build_apps_v1beta1_kubeclient(MINIKUBE_CONTEXT) + @apps_v1beta1_kubeclient ||= build_apps_v1beta1_kubeclient(TEST_CONTEXT) end def batch_v1beta1_kubeclient - @batch_v1beta1_kubeclient ||= build_batch_v1beta1_kubeclient(MINIKUBE_CONTEXT) + @batch_v1beta1_kubeclient ||= build_batch_v1beta1_kubeclient(TEST_CONTEXT) end def batch_v1_kubeclient - @batch_v1_kubeclient ||= build_batch_v1_kubeclient(MINIKUBE_CONTEXT) + @batch_v1_kubeclient ||= build_batch_v1_kubeclient(TEST_CONTEXT) end def apiextensions_v1beta1_kubeclient - @apiextensions_v1beta1_kubeclient ||= build_apiextensions_v1beta1_kubeclient(MINIKUBE_CONTEXT) + @apiextensions_v1beta1_kubeclient ||= build_apiextensions_v1beta1_kubeclient(TEST_CONTEXT) end def autoscaling_v1_kubeclient - @autoscaling_v1_kubeclient ||= build_autoscaling_v1_kubeclient(MINIKUBE_CONTEXT) + @autoscaling_v1_kubeclient ||= build_autoscaling_v1_kubeclient(TEST_CONTEXT) end end diff --git a/test/helpers/task_runner_test_helper.rb b/test/helpers/task_runner_test_helper.rb index 4b31a632c..3c12b0dba 100644 --- a/test/helpers/task_runner_test_helper.rb +++ b/test/helpers/task_runner_test_helper.rb @@ -15,7 +15,7 @@ def deploy_task_template(subset = ["template-runner.yml", "configmap-data.yml"]) end def build_task_runner(ns: @namespace, max_watch_seconds: nil) - KubernetesDeploy::RunnerTask.new(context: KubeclientHelper::MINIKUBE_CONTEXT, namespace: ns, logger: logger, + KubernetesDeploy::RunnerTask.new(context: KubeclientHelper::TEST_CONTEXT, namespace: ns, logger: logger, max_watch_seconds: max_watch_seconds) end diff --git a/test/integration-serial/serial_deploy_test.rb b/test/integration-serial/serial_deploy_test.rb index c606213b2..6fdff9e07 100644 --- a/test/integration-serial/serial_deploy_test.rb +++ b/test/integration-serial/serial_deploy_test.rb @@ -70,7 +70,7 @@ def test_invalid_context 'Unable to connect to the server', 'Unable to connect to the server', 'Result: FAILURE', - 'Failed to reach server for minikube', + "Failed to reach server for #{TEST_CONTEXT}", ], in_order: true) ensure ENV['KUBECONFIG'] = old_config diff --git a/test/integration/kubernetes_deploy_test.rb b/test/integration/kubernetes_deploy_test.rb index 001acbdd6..5c01dcd2c 100644 --- a/test/integration/kubernetes_deploy_test.rb +++ b/test/integration/kubernetes_deploy_test.rb @@ -15,11 +15,12 @@ def test_full_hello_cloud_set_deploy_succeeds "Successfully deployed 19 resources" ], in_order: true) + num_ds = expected_daemonset_pod_count assert_logs_match_all([ %r{ReplicaSet/bare-replica-set\s+1 replica, 1 availableReplica, 1 readyReplica}, %r{Deployment/web\s+1 replica, 1 updatedReplica, 1 availableReplica}, %r{Service/web\s+Selects at least 1 pod}, - %r{DaemonSet/ds-app\s+1 updatedNumberScheduled, 1 desiredNumberScheduled, 1 numberReady}, + %r[DaemonSet/ds-app\s+#{num_ds} updatedNumberScheduled, #{num_ds} desiredNumberScheduled, #{num_ds} numberReady], %r{StatefulSet/stateful-busybox}, %r{Service/redis-external\s+Doesn't require any endpoint}, "- Job/hello-job (timeout: 600s)", @@ -771,11 +772,12 @@ def test_output_when_unmanaged_pod_preexists def test_bad_container_on_daemon_sets_fails assert_deploy_failure(deploy_fixtures("invalid", subset: ["crash_loop_daemon_set.yml"])) + num_ds = expected_daemonset_pod_count assert_logs_match_all([ "Failed to deploy 1 resource", "DaemonSet/crash-loop: FAILED", "crash-loop-back-off: Crashing repeatedly (exit 1). See logs for more information.", - "Final status: 1 updatedNumberScheduled, 1 desiredNumberScheduled, 0 numberReady", + "Final status: #{num_ds} updatedNumberScheduled, #{num_ds} desiredNumberScheduled, 0 numberReady", "Events (common success events excluded):", "BackOff: Back-off restarting failed container", "Logs from container 'crash-loop-back-off':", @@ -1014,7 +1016,7 @@ def test_friendly_error_on_misidentified_erb_file end def test_adds_namespace_labels_to_statsd_tags - desired_tags = %W(context:#{KubeclientHelper::MINIKUBE_CONTEXT} namespace:#{@namespace} foo:bar) + desired_tags = %W(context:#{KubeclientHelper::TEST_CONTEXT} namespace:#{@namespace} foo:bar) hello_cloud = FixtureSetAssertions::HelloCloud.new(@namespace) kubeclient.patch_namespace(hello_cloud.namespace, metadata: { labels: { foo: 'bar' } }) metrics = capture_statsd_calls do @@ -1078,4 +1080,14 @@ def test_not_apply_resource_can_be_pruned /The following resources were pruned: #{pod_disruption_budget_matcher}/ ]) end + + private + + def expected_daemonset_pod_count + nodes = kubeclient.get_nodes + return 1 if nodes.one? + nodes.count do |node| + !node.metadata.labels.to_h.keys.include?(:"node-role.kubernetes.io/master") + end + end end diff --git a/test/integration/restart_task_test.rb b/test/integration/restart_task_test.rb index 9c27e12e8..7169d53a2 100644 --- a/test/integration/restart_task_test.rb +++ b/test/integration/restart_task_test.rb @@ -141,14 +141,14 @@ def test_restart_not_existing_context def test_restart_not_existing_namespace restart = KubernetesDeploy::RestartTask.new( - context: KubeclientHelper::MINIKUBE_CONTEXT, + context: KubeclientHelper::TEST_CONTEXT, namespace: "walrus", logger: logger ) assert_restart_failure(restart.perform(["web"])) assert_logs_match_all([ "Result: FAILURE", - "Namespace `walrus` not found in context `minikube`" + "Namespace `walrus` not found in context `#{TEST_CONTEXT}`" ], in_order: true) end @@ -224,7 +224,7 @@ def test_restart_successful_with_partial_availability def build_restart_task KubernetesDeploy::RestartTask.new( - context: KubeclientHelper::MINIKUBE_CONTEXT, + context: KubeclientHelper::TEST_CONTEXT, namespace: @namespace, logger: logger ) diff --git a/test/integration/runner_task_test.rb b/test/integration/runner_task_test.rb index 2bde99948..5d6487d39 100644 --- a/test/integration/runner_task_test.rb +++ b/test/integration/runner_task_test.rb @@ -195,7 +195,7 @@ def test_run_with_template_missing task_runner = build_task_runner assert_task_run_failure(task_runner.run(run_params)) message = "Pod template `hello-cloud-template-runner` not found in namespace `#{@namespace}`, " \ - "context `#{KubeclientHelper::MINIKUBE_CONTEXT}`" + "context `#{KubeclientHelper::TEST_CONTEXT}`" assert_logs_match_all([ "Result: FAILURE", message diff --git a/test/test_helper.rb b/test/test_helper.rb index e5b95d5dd..c89d8284e 100644 --- a/test/test_helper.rb +++ b/test/test_helper.rb @@ -88,7 +88,7 @@ def setup def logger @logger ||= begin device = log_to_stderr? ? $stderr : @logger_stream - KubernetesDeploy::FormattedLogger.build(@namespace, KubeclientHelper::MINIKUBE_CONTEXT, device) + KubernetesDeploy::FormattedLogger.build(@namespace, KubeclientHelper::TEST_CONTEXT, device) end end @@ -208,7 +208,7 @@ def stub_kubectl_response(*args, resp:, err: "", raise_if_not_found: nil, succes end def build_runless_kubectl - obj = KubernetesDeploy::Kubectl.new(namespace: 'test', context: KubeclientHelper::MINIKUBE_CONTEXT, + obj = KubernetesDeploy::Kubectl.new(namespace: 'test', context: KubeclientHelper::TEST_CONTEXT, logger: logger, log_failure_by_default: false) def obj.run(*) ["", "", SystemExit.new(0)] @@ -248,8 +248,8 @@ def prune_matcher(kind, group, name) kind + '(.' + group + ')?[ \/]"?' + name + '"?' end - logger = KubernetesDeploy::FormattedLogger.build("default", MINIKUBE_CONTEXT, $stderr) - kubectl = KubernetesDeploy::Kubectl.new(namespace: "default", context: MINIKUBE_CONTEXT, logger: logger, + logger = KubernetesDeploy::FormattedLogger.build("default", TEST_CONTEXT, $stderr) + kubectl = KubernetesDeploy::Kubectl.new(namespace: "default", context: TEST_CONTEXT, logger: logger, log_failure_by_default: true, default_timeout: '5s') KUBE_CLIENT_VERSION = kubectl.client_version @@ -301,8 +301,8 @@ def self.prepare_pv(name) def self.deploy_metric_server # Set-up the metric server that the HPA needs https://github.com/kubernetes-incubator/metrics-server - logger = KubernetesDeploy::FormattedLogger.build("default", KubeclientHelper::MINIKUBE_CONTEXT, $stderr) - kubectl = KubernetesDeploy::Kubectl.new(namespace: "kube-system", context: KubeclientHelper::MINIKUBE_CONTEXT, + logger = KubernetesDeploy::FormattedLogger.build("default", KubeclientHelper::TEST_CONTEXT, $stderr) + kubectl = KubernetesDeploy::Kubectl.new(namespace: "kube-system", context: KubeclientHelper::TEST_CONTEXT, logger: logger, log_failure_by_default: true, default_timeout: '5s') Dir.glob("test/setup/metrics-server/*.{yml,yaml}*").map do |resource| diff --git a/test/unit/kubernetes-deploy/ejson_secret_provisioner_test.rb b/test/unit/kubernetes-deploy/ejson_secret_provisioner_test.rb index bbc4d9e57..13477beb3 100644 --- a/test/unit/kubernetes-deploy/ejson_secret_provisioner_test.rb +++ b/test/unit/kubernetes-deploy/ejson_secret_provisioner_test.rb @@ -139,7 +139,7 @@ def build_provisioner(dir = nil) dir ||= fixture_path('ejson-cloud') KubernetesDeploy::EjsonSecretProvisioner.new( namespace: 'test', - context: KubeclientHelper::MINIKUBE_CONTEXT, + context: KubeclientHelper::TEST_CONTEXT, template_dir: dir, logger: logger ) diff --git a/test/unit/kubernetes-deploy/kubeclient_builder_test.rb b/test/unit/kubernetes-deploy/kubeclient_builder_test.rb index f93711748..02ac126f3 100644 --- a/test/unit/kubernetes-deploy/kubeclient_builder_test.rb +++ b/test/unit/kubernetes-deploy/kubeclient_builder_test.rb @@ -23,7 +23,7 @@ def test_build_client_from_multiple_config_files build_v1_kubeclient(context_name) end # Build kubeclient for an existing context success - context_name = KubeclientHelper::MINIKUBE_CONTEXT + context_name = KubeclientHelper::TEST_CONTEXT client = build_v1_kubeclient(context_name) assert !client.nil?, "Expected Kubeclient is built for context " \ "#{context_name} with success." diff --git a/test/unit/kubernetes-deploy/runner_task_test.rb b/test/unit/kubernetes-deploy/runner_task_test.rb index b103b6001..4c4a7dc25 100644 --- a/test/unit/kubernetes-deploy/runner_task_test.rb +++ b/test/unit/kubernetes-deploy/runner_task_test.rb @@ -5,7 +5,7 @@ class RunnerTaskUnitTest < KubernetesDeploy::TestCase def test_run_with_invalid_configuration task_runner = KubernetesDeploy::RunnerTask.new( - context: KubeclientHelper::MINIKUBE_CONTEXT, + context: KubeclientHelper::TEST_CONTEXT, namespace: nil, logger: logger, ) @@ -18,7 +18,7 @@ def test_run_with_invalid_configuration def test_run_bang_with_invalid_configuration task_runner = KubernetesDeploy::RunnerTask.new( - context: KubeclientHelper::MINIKUBE_CONTEXT, + context: KubeclientHelper::TEST_CONTEXT, namespace: nil, logger: logger, ) diff --git a/test/unit/sync_mediator_test.rb b/test/unit/sync_mediator_test.rb index 0f5fe86b9..9680db920 100644 --- a/test/unit/sync_mediator_test.rb +++ b/test/unit/sync_mediator_test.rb @@ -156,7 +156,7 @@ def test_sync_uses_kubectl_resource_type private def mediator - @mediator ||= KubernetesDeploy::SyncMediator.new(namespace: 'test-ns', context: KubeclientHelper::MINIKUBE_CONTEXT, + @mediator ||= KubernetesDeploy::SyncMediator.new(namespace: 'test-ns', context: KubeclientHelper::TEST_CONTEXT, logger: logger) end From bff9424d26fe3fe183a08c3e3ff2c5f376e37929 Mon Sep 17 00:00:00 2001 From: Katrina Verey Date: Fri, 26 Oct 2018 12:06:08 -0400 Subject: [PATCH 2/2] Rubocop --- test/integration/kubernetes_deploy_test.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/integration/kubernetes_deploy_test.rb b/test/integration/kubernetes_deploy_test.rb index 5c01dcd2c..f7a18c7d1 100644 --- a/test/integration/kubernetes_deploy_test.rb +++ b/test/integration/kubernetes_deploy_test.rb @@ -20,7 +20,7 @@ def test_full_hello_cloud_set_deploy_succeeds %r{ReplicaSet/bare-replica-set\s+1 replica, 1 availableReplica, 1 readyReplica}, %r{Deployment/web\s+1 replica, 1 updatedReplica, 1 availableReplica}, %r{Service/web\s+Selects at least 1 pod}, - %r[DaemonSet/ds-app\s+#{num_ds} updatedNumberScheduled, #{num_ds} desiredNumberScheduled, #{num_ds} numberReady], + %r{DaemonSet/ds-app\s+#{num_ds} updatedNumberScheduled, #{num_ds} desiredNumberScheduled, #{num_ds} numberReady}, %r{StatefulSet/stateful-busybox}, %r{Service/redis-external\s+Doesn't require any endpoint}, "- Job/hello-job (timeout: 600s)",