Support test clusters other than minikube

.gitignore

@@ -17,3 +17,4 @@ coverage/*
 dev/flamegraph.svg
 dev/profile
 dev/flamegraph.pl
+.local-context

README.md

@@ -385,16 +385,23 @@ To install this gem onto your local machine, run `bundle exec rake install`.
 
 ## Running the test suite locally
 
-1. Start [minikube](https://kubernetes.io/docs/getting-started-guides/minikube/#installation) (`minikube start [options]`)
-2. Make sure you have a context named "minikube" in your kubeconfig. Minikube adds this context for you when you run `minikube start`; please do not rename it. You can check for it using `kubectl config get-contexts`.
-3. Run `bundle exec rake test`
+Using minikube:
 
+1. Start [minikube](https://kubernetes.io/docs/getting-started-guides/minikube/#installation) (`minikube start [options]`).
+2. Make sure you have a context named "minikube" in your kubeconfig. Minikube adds this context for you when you run `minikube start`. You can check for it using `kubectl config get-contexts`.
+3. Run `bundle exec rake test` (or `dev test` if you work for Shopify).
 
+Using another local cluster:
 
-To see the full-color output of a specific integration test, you can use `PRINT_LOGS=1 bundle exec ruby -I test test/integration/kubernetes_deploy_test.rb -n/test_name/`.
+1. Start your cluster.
+2. Put the name of the context you want to use in a file named `.local-context` in the root of this project. For example: `echo "dind" > .local-context`.
+3. Run `bundle exec rake test` (or `dev test` if you work for Shopify).
 
 To make StatsD log what it would have emitted, run a test with `STATSD_DEV=1`.
 
+To see the full-color output of a specific integration test, you can use `PRINT_LOGS=1`. For example: `PRINT_LOGS=1 bundle exec ruby -I test test/integration/kubernetes_deploy_test.rb -n/test_name/`.
+
+
 
 
 ![test-output](screenshots/test-output.png)

test/fixtures/kube-config/invalid_config.yml

@@ -3,18 +3,23 @@ clusters:
 - cluster:
     insecure-skip-tls-verify: true
     server: https://192.168.5.1
-  name: minikube
+  name: invalid
 contexts:
 - context:
-    cluster: minikube
-    namespace: default
-    user: minikube
+    cluster: invalid
+    user: test-user
   name: minikube
-current-context: ""
+- context:
+    cluster: invalid
+    user: test-user
+  name: dind
+- context:
+    cluster: invalid
+    user: test-user
+  name: docker-for-mac
 kind: Config
-preferences: {}
 users:
-- name: minikube
+- name: test-user
   user:
     username: username
     password: password

test/helpers/fixture_deploy_helper.rb

@@ -68,7 +68,7 @@ def deploy_dir_without_profiling(dir, wait: true, allow_protected_ns: false, pru
     deploy = KubernetesDeploy::DeployTask.new(
       namespace: @namespace,
       current_sha: current_sha,
-      context: KubeclientHelper::MINIKUBE_CONTEXT,
+      context: KubeclientHelper::TEST_CONTEXT,
       template_dir: dir,
       logger: logger,
       kubectl_instance: kubectl_instance,
@@ -129,7 +129,7 @@ def write_fixtures_to_dir(fixtures, target_dir)
   end
 
   def build_kubectl(log_failure_by_default: true, timeout: '5s')
-    KubernetesDeploy::Kubectl.new(namespace: @namespace, context: KubeclientHelper::MINIKUBE_CONTEXT, logger: logger,
+    KubernetesDeploy::Kubectl.new(namespace: @namespace, context: KubeclientHelper::TEST_CONTEXT, logger: logger,
       log_failure_by_default: log_failure_by_default, default_timeout: timeout)
   end
 end

test/helpers/kubeclient_helper.rb

@@ -2,39 +2,43 @@
 require 'kubernetes-deploy/kubeclient_builder'
 
 module KubeclientHelper
-  MINIKUBE_CONTEXT = "minikube"
+  LOCAL_CONTEXT_OVERRIDE_PATH = File.expand_path("../../.local-context", __dir__)
+  if File.exist?(LOCAL_CONTEXT_OVERRIDE_PATH)
+    TEST_CONTEXT = File.read(LOCAL_CONTEXT_OVERRIDE_PATH).split.first
+  end
+  TEST_CONTEXT ||= "minikube"
 
   include KubernetesDeploy::KubeclientBuilder
 
   def kubeclient
-    @kubeclient ||= build_v1_kubeclient(MINIKUBE_CONTEXT)
+    @kubeclient ||= build_v1_kubeclient(TEST_CONTEXT)
   end
 
   def v1beta1_kubeclient
-    @v1beta1_kubeclient ||= build_v1beta1_kubeclient(MINIKUBE_CONTEXT)
+    @v1beta1_kubeclient ||= build_v1beta1_kubeclient(TEST_CONTEXT)
   end
 
   def policy_v1beta1_kubeclient
-    @policy_v1beta1_kubeclient ||= build_policy_v1beta1_kubeclient(MINIKUBE_CONTEXT)
+    @policy_v1beta1_kubeclient ||= build_policy_v1beta1_kubeclient(TEST_CONTEXT)
   end
 
   def apps_v1beta1_kubeclient
-    @apps_v1beta1_kubeclient ||= build_apps_v1beta1_kubeclient(MINIKUBE_CONTEXT)
+    @apps_v1beta1_kubeclient ||= build_apps_v1beta1_kubeclient(TEST_CONTEXT)
   end
 
   def batch_v1beta1_kubeclient
-    @batch_v1beta1_kubeclient ||= build_batch_v1beta1_kubeclient(MINIKUBE_CONTEXT)
+    @batch_v1beta1_kubeclient ||= build_batch_v1beta1_kubeclient(TEST_CONTEXT)
   end
 
   def batch_v1_kubeclient
-    @batch_v1_kubeclient ||= build_batch_v1_kubeclient(MINIKUBE_CONTEXT)
+    @batch_v1_kubeclient ||= build_batch_v1_kubeclient(TEST_CONTEXT)
   end
 
   def apiextensions_v1beta1_kubeclient
-    @apiextensions_v1beta1_kubeclient ||= build_apiextensions_v1beta1_kubeclient(MINIKUBE_CONTEXT)
+    @apiextensions_v1beta1_kubeclient ||= build_apiextensions_v1beta1_kubeclient(TEST_CONTEXT)
   end
 
   def autoscaling_v1_kubeclient
-    @autoscaling_v1_kubeclient ||= build_autoscaling_v1_kubeclient(MINIKUBE_CONTEXT)
+    @autoscaling_v1_kubeclient ||= build_autoscaling_v1_kubeclient(TEST_CONTEXT)
   end
 end

test/helpers/task_runner_test_helper.rb

@@ -15,7 +15,7 @@ def deploy_task_template(subset = ["template-runner.yml", "configmap-data.yml"])
   end
 
   def build_task_runner(ns: @namespace, max_watch_seconds: nil)
-    KubernetesDeploy::RunnerTask.new(context: KubeclientHelper::MINIKUBE_CONTEXT, namespace: ns, logger: logger,
+    KubernetesDeploy::RunnerTask.new(context: KubeclientHelper::TEST_CONTEXT, namespace: ns, logger: logger,
       max_watch_seconds: max_watch_seconds)
   end
 

test/integration-serial/serial_deploy_test.rb

@@ -70,7 +70,7 @@ def test_invalid_context
         'Unable to connect to the server',
         'Unable to connect to the server',
         'Result: FAILURE',
-        'Failed to reach server for minikube',
+        "Failed to reach server for #{TEST_CONTEXT}",
       ], in_order: true)
     ensure
       ENV['KUBECONFIG'] = old_config

test/integration/kubernetes_deploy_test.rb

@@ -15,11 +15,12 @@ def test_full_hello_cloud_set_deploy_succeeds
       "Successfully deployed 19 resources"
     ], in_order: true)
 
+    num_ds = expected_daemonset_pod_count
     assert_logs_match_all([
       %r{ReplicaSet/bare-replica-set\s+1 replica, 1 availableReplica, 1 readyReplica},
       %r{Deployment/web\s+1 replica, 1 updatedReplica, 1 availableReplica},
       %r{Service/web\s+Selects at least 1 pod},
-      %r{DaemonSet/ds-app\s+1 updatedNumberScheduled, 1 desiredNumberScheduled, 1 numberReady},
+      %r{DaemonSet/ds-app\s+#{num_ds} updatedNumberScheduled, #{num_ds} desiredNumberScheduled, #{num_ds} numberReady},
       %r{StatefulSet/stateful-busybox},
       %r{Service/redis-external\s+Doesn't require any endpoint},
       "- Job/hello-job (timeout: 600s)",
@@ -771,11 +772,12 @@ def test_output_when_unmanaged_pod_preexists
 
   def test_bad_container_on_daemon_sets_fails
     assert_deploy_failure(deploy_fixtures("invalid", subset: ["crash_loop_daemon_set.yml"]))
+    num_ds = expected_daemonset_pod_count
     assert_logs_match_all([
       "Failed to deploy 1 resource",
       "DaemonSet/crash-loop: FAILED",
       "crash-loop-back-off: Crashing repeatedly (exit 1). See logs for more information.",
-      "Final status: 1 updatedNumberScheduled, 1 desiredNumberScheduled, 0 numberReady",
+      "Final status: #{num_ds} updatedNumberScheduled, #{num_ds} desiredNumberScheduled, 0 numberReady",
       "Events (common success events excluded):",
       "BackOff: Back-off restarting failed container",
       "Logs from container 'crash-loop-back-off':",
@@ -1014,7 +1016,7 @@ def test_friendly_error_on_misidentified_erb_file
   end
 
   def test_adds_namespace_labels_to_statsd_tags
-    desired_tags = %W(context:#{KubeclientHelper::MINIKUBE_CONTEXT} namespace:#{@namespace} foo:bar)
+    desired_tags = %W(context:#{KubeclientHelper::TEST_CONTEXT} namespace:#{@namespace} foo:bar)
     hello_cloud = FixtureSetAssertions::HelloCloud.new(@namespace)
     kubeclient.patch_namespace(hello_cloud.namespace, metadata: { labels: { foo: 'bar' } })
     metrics = capture_statsd_calls do
@@ -1078,4 +1080,14 @@ def test_not_apply_resource_can_be_pruned
       /The following resources were pruned: #{pod_disruption_budget_matcher}/
     ])
   end
+
+  private
+
+  def expected_daemonset_pod_count
+    nodes = kubeclient.get_nodes
+    return 1 if nodes.one?
+    nodes.count do |node|
+      !node.metadata.labels.to_h.keys.include?(:"node-role.kubernetes.io/master")
+    end
+  end
 end

test/integration/restart_task_test.rb

@@ -141,14 +141,14 @@ def test_restart_not_existing_context
 
   def test_restart_not_existing_namespace
     restart = KubernetesDeploy::RestartTask.new(
-      context: KubeclientHelper::MINIKUBE_CONTEXT,
+      context: KubeclientHelper::TEST_CONTEXT,
       namespace: "walrus",
       logger: logger
     )
     assert_restart_failure(restart.perform(["web"]))
     assert_logs_match_all([
       "Result: FAILURE",
-      "Namespace `walrus` not found in context `minikube`"
+      "Namespace `walrus` not found in context `#{TEST_CONTEXT}`"
     ],
       in_order: true)
   end
@@ -224,7 +224,7 @@ def test_restart_successful_with_partial_availability
 
   def build_restart_task
     KubernetesDeploy::RestartTask.new(
-      context: KubeclientHelper::MINIKUBE_CONTEXT,
+      context: KubeclientHelper::TEST_CONTEXT,
       namespace: @namespace,
       logger: logger
     )

test/integration/runner_task_test.rb

@@ -195,7 +195,7 @@ def test_run_with_template_missing
     task_runner = build_task_runner
     assert_task_run_failure(task_runner.run(run_params))
     message = "Pod template `hello-cloud-template-runner` not found in namespace `#{@namespace}`, " \
-      "context `#{KubeclientHelper::MINIKUBE_CONTEXT}`"
+      "context `#{KubeclientHelper::TEST_CONTEXT}`"
     assert_logs_match_all([
       "Result: FAILURE",
       message

test/test_helper.rb

@@ -88,7 +88,7 @@ def setup
     def logger
       @logger ||= begin
         device = log_to_stderr? ? $stderr : @logger_stream
-        KubernetesDeploy::FormattedLogger.build(@namespace, KubeclientHelper::MINIKUBE_CONTEXT, device)
+        KubernetesDeploy::FormattedLogger.build(@namespace, KubeclientHelper::TEST_CONTEXT, device)
       end
     end
 
@@ -208,7 +208,7 @@ def stub_kubectl_response(*args, resp:, err: "", raise_if_not_found: nil, succes
     end
 
     def build_runless_kubectl
-      obj = KubernetesDeploy::Kubectl.new(namespace: 'test', context: KubeclientHelper::MINIKUBE_CONTEXT,
+      obj = KubernetesDeploy::Kubectl.new(namespace: 'test', context: KubeclientHelper::TEST_CONTEXT,
         logger: logger, log_failure_by_default: false)
       def obj.run(*)
         ["", "", SystemExit.new(0)]
@@ -248,8 +248,8 @@ def prune_matcher(kind, group, name)
       kind + '(.' + group + ')?[ \/]"?' + name + '"?'
     end
 
-    logger = KubernetesDeploy::FormattedLogger.build("default", MINIKUBE_CONTEXT, $stderr)
-    kubectl = KubernetesDeploy::Kubectl.new(namespace: "default", context: MINIKUBE_CONTEXT, logger: logger,
+    logger = KubernetesDeploy::FormattedLogger.build("default", TEST_CONTEXT, $stderr)
+    kubectl = KubernetesDeploy::Kubectl.new(namespace: "default", context: TEST_CONTEXT, logger: logger,
                                             log_failure_by_default: true, default_timeout: '5s')
 
     KUBE_CLIENT_VERSION = kubectl.client_version
@@ -301,8 +301,8 @@ def self.prepare_pv(name)
 
     def self.deploy_metric_server
       # Set-up the metric server that the HPA needs https://github.com/kubernetes-incubator/metrics-server
-      logger = KubernetesDeploy::FormattedLogger.build("default", KubeclientHelper::MINIKUBE_CONTEXT, $stderr)
-      kubectl = KubernetesDeploy::Kubectl.new(namespace: "kube-system", context: KubeclientHelper::MINIKUBE_CONTEXT,
+      logger = KubernetesDeploy::FormattedLogger.build("default", KubeclientHelper::TEST_CONTEXT, $stderr)
+      kubectl = KubernetesDeploy::Kubectl.new(namespace: "kube-system", context: KubeclientHelper::TEST_CONTEXT,
         logger: logger, log_failure_by_default: true, default_timeout: '5s')
 
       Dir.glob("test/setup/metrics-server/*.{yml,yaml}*").map do |resource|

test/unit/kubernetes-deploy/ejson_secret_provisioner_test.rb

@@ -139,7 +139,7 @@ def build_provisioner(dir = nil)
     dir ||= fixture_path('ejson-cloud')
     KubernetesDeploy::EjsonSecretProvisioner.new(
       namespace: 'test',
-      context: KubeclientHelper::MINIKUBE_CONTEXT,
+      context: KubeclientHelper::TEST_CONTEXT,
       template_dir: dir,
       logger: logger
     )

test/unit/kubernetes-deploy/kubeclient_builder_test.rb

@@ -23,7 +23,7 @@ def test_build_client_from_multiple_config_files
       build_v1_kubeclient(context_name)
     end
     # Build kubeclient for an existing context success
-    context_name = KubeclientHelper::MINIKUBE_CONTEXT
+    context_name = KubeclientHelper::TEST_CONTEXT
     client = build_v1_kubeclient(context_name)
     assert !client.nil?, "Expected Kubeclient is built for context " \
     	"#{context_name} with success."

test/unit/kubernetes-deploy/runner_task_test.rb

@@ -5,7 +5,7 @@
 class RunnerTaskUnitTest < KubernetesDeploy::TestCase
   def test_run_with_invalid_configuration
     task_runner = KubernetesDeploy::RunnerTask.new(
-      context: KubeclientHelper::MINIKUBE_CONTEXT,
+      context: KubeclientHelper::TEST_CONTEXT,
       namespace: nil,
       logger: logger,
     )
@@ -18,7 +18,7 @@ def test_run_with_invalid_configuration
 
   def test_run_bang_with_invalid_configuration
     task_runner = KubernetesDeploy::RunnerTask.new(
-      context: KubeclientHelper::MINIKUBE_CONTEXT,
+      context: KubeclientHelper::TEST_CONTEXT,
       namespace: nil,
       logger: logger,
     )

test/unit/sync_mediator_test.rb

@@ -156,7 +156,7 @@ def test_sync_uses_kubectl_resource_type
   private
 
   def mediator
-    @mediator ||= KubernetesDeploy::SyncMediator.new(namespace: 'test-ns', context: KubeclientHelper::MINIKUBE_CONTEXT,
+    @mediator ||= KubernetesDeploy::SyncMediator.new(namespace: 'test-ns', context: KubeclientHelper::TEST_CONTEXT,
       logger: logger)
   end