Merge pull request zendesk#1636 from zendesk/grosser/fork

use forking test runner to get rid of random test errors

Gemfile

@@ -124,6 +124,7 @@ group :development, :test do
   gem 'rubocop'
   gem 'flay'
   gem 'parallel_tests'
+  gem 'forking_test_runner'
 end
 
 group :test do

Gemfile.lock

@@ -206,6 +206,9 @@ GEM
     flowdock (0.6.0)
       httparty (~> 0.7)
       multi_json
+    forking_test_runner (0.8.0)
+      activerecord (< 5.1.0)
+      parallel_tests (>= 1.3.7)
     github-markdown (0.6.9)
     globalid (0.3.7)
       activesupport (>= 4.1.0)
@@ -526,6 +529,7 @@ DEPENDENCIES
   faraday
   faraday-http-cache
   flay
+  forking_test_runner
   github-markdown
   goldiloader
   kaminari (~> 0.17.0)

Rakefile

@@ -28,17 +28,9 @@ task :asset_compilation_environment do
 end
 Rake::Task['assets:precompile'].prerequisites.unshift :asset_compilation_environment
 
-namespace :plugins do
-  Rake::TestTask.new(:test) do |t|
-    t.pattern = "plugins/*/test/**/*_test.rb"
-    t.warning = false
-  end
-end
-
 Rake::Task['test'].clear
-Rake::TestTask.new(:test) do |t|
-  t.pattern = "{test,plugins/*/test}/**/*_test.rb"
-  t.warning = false
+task :test do
+  sh "forking-test-runner test plugins/*/test --merge-coverage --quiet"
 end
 
 namespace :test do

app/controllers/admin/deploy_groups_controller.rb

@@ -19,7 +19,6 @@ def show
 
   def new
     @deploy_group = DeployGroup.new
-    Samson::Hooks.fire(:edit_deploy_group, @deploy_group)
     render :edit
   end
 
@@ -34,7 +33,6 @@ def create
   end
 
   def edit
-    Samson::Hooks.fire(:edit_deploy_group, @deploy_group)
   end
 
   def update

lib/samson/hooks.rb

@@ -36,7 +36,6 @@ class UserError < StandardError
       :after_docker_build,
       :after_job_execution,
       :job_additional_vars,
-      :edit_deploy_group,
       :buildkite_release_params,
       :release_deploy_conditions,
       :deploy_group_env

plugins/airbrake/test/samson_airbrake/samson_plugin_test.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 require_relative '../test_helper'
 
-SingleCov.covered! unless defined?(Rake) # rake preloads all plugins
+SingleCov.covered!
 
 describe SamsonAirbrake::Engine do
   describe :after_deploy do

plugins/aws_ecr/test/samson_aws_ecr/samson_plugin_test.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 require_relative '../test_helper'
 
-SingleCov.covered! unless defined?(Rake) # rake preloads all plugins
+SingleCov.covered!
 
 describe SamsonAwsEcr::Engine do
   def clear_client

plugins/datadog/test/samson_datadog/samson_plugin_test.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 require_relative '../test_helper'
 
-SingleCov.covered! unless defined?(Rake) # rake preloads all plugins
+SingleCov.covered!
 
 describe SamsonDatadog do
   let(:deploy) { deploys(:succeeded_test) }

plugins/docker_binary_builder/test/samson_docker_binary_builder/samson_plugin_test.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 require_relative '../test_helper'
 
-SingleCov.covered! uncovered: 2 unless defined?(Rake) # rake preloads all plugins
+SingleCov.covered! uncovered: 2
 
 describe SamsonDockerBinaryBuilder do
   describe '#after_deploy_setup' do

plugins/dockerb/test/samson_dockerb/samson_plugin_test.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 require_relative '../test_helper'
 
-SingleCov.covered! unless defined?(Rake) # rake preloads all plugins
+SingleCov.covered!
 
 describe SamsonDockerb do
   let(:stage) { stages(:test_staging) }

plugins/env/test/samson_env/samson_plugin_test.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 require_relative '../test_helper'
 
-SingleCov.covered! unless defined?(Rake) # rake preloads all plugins
+SingleCov.covered!
 
 describe SamsonEnv do
   let(:deploy) { deploys(:succeeded_test) }

plugins/flowdock/test/samson_flowdock/samson_plugin_test.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 require_relative '../test_helper'
 
-SingleCov.covered! uncovered: 1 unless defined?(Rake) # rake preloads all plugins
+SingleCov.covered! uncovered: 1
 
 describe SamsonFlowdock do
   let(:deploy) { deploys(:succeeded_test) }

plugins/hyperclair/test/samson_hyperclair/samson_plugin_test.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 require_relative '../test_helper'
 
-SingleCov.covered! unless defined?(Rake) # rake preloads all plugins
+SingleCov.covered!
 
 describe SamsonHyperclair do
   describe :after_docker_build do

plugins/jenkins/test/samson_jenkins/samson_plugin_test.rb

@@ -1,11 +1,17 @@
 # frozen_string_literal: true
 require_relative "../test_helper"
 
-SingleCov.covered! unless defined?(Rake) # rake preloads all plugins
+SingleCov.covered!
 
 describe SamsonJenkins do
   it "calls deployed! on deploy" do
     Samson::Jenkins.expects(:deployed!)
     Samson::Hooks.fire :after_deploy, deploys(:succeeded_test), users(:admin)
   end
+
+  describe :stage_permitted_params do
+    it "adds parameters" do
+      Samson::Hooks.fire(:stage_permitted_params).flatten.must_include :jenkins_job_names
+    end
+  end
 end

plugins/kubernetes/app/views/samson_kubernetes/_deploy_group_form.html.erb

@@ -1,3 +1,4 @@
+<% form.object.cluster_deploy_group || form.object.build_cluster_deploy_group %>
 <%= form.fields_for :cluster_deploy_group, include_id: false do |cluster_form| %>
   <%= cluster_form.input :kubernetes_cluster_id, label: "Kubernetes Cluster" do %>
     <%= cluster_form.collection_select(:kubernetes_cluster_id, Kubernetes::Cluster.all, :id, :name, { include_blank: true }, { class: 'form-control' }) %>

plugins/kubernetes/lib/samson_kubernetes/samson_plugin.rb

@@ -27,7 +27,3 @@ class Engine < Rails::Engine
 Samson::Hooks.callback(:stage_permitted_params) { :kubernetes }
 Samson::Hooks.callback(:deploy_permitted_params) { [:kubernetes_rollback, :kubernetes_reuse_build] }
 Samson::Hooks.callback(:build_permitted_params) { :kubernetes_job }
-
-Samson::Hooks.callback :edit_deploy_group do |deploy_group|
-  deploy_group.build_cluster_deploy_group unless deploy_group.cluster_deploy_group
-end

plugins/kubernetes/test/models/kubernetes/deploy_executor_test.rb

@@ -150,14 +150,12 @@ def worker_is_unstable
 
       it "fails before building when roles are invalid" do
         Kubernetes::ReleaseDoc.any_instance.unstub(:raw_template)
-        GitRepository.any_instance.expects(:file_content).
-          with('kubernetes/resque_worker.yml', anything).
-          returns("oops: bad")
+        GitRepository.any_instance.expects(:file_content).with { |file| file =~ /^kubernetes\// }.returns("oops: bad")
 
         e = assert_raises Samson::Hooks::UserError do
           refute execute!
         end
-        e.message.must_include "Error found when parsing kubernetes/resque_worker.yml"
+        e.message.must_include "Error found when parsing kubernetes/"
       end
 
       it "fails before building when secrets are not configured in the backend" do
@@ -177,7 +175,7 @@ def worker_is_unstable
     describe "role settings" do
       it "uses configured role settings" do
         assert execute!
-        doc = Kubernetes::Release.last.release_docs.sort_by(&:id).last
+        doc = Kubernetes::Release.last.release_docs.sort_by(&:kubernetes_role).last
         config = server_role
         doc.replica_target.must_equal config.replicas
         doc.cpu.must_equal config.cpu

plugins/kubernetes/test/samson_kubernetes/samson_plugin_test.rb

@@ -1,26 +1,33 @@
 # frozen_string_literal: true
 require_relative "../test_helper"
 
-SingleCov.covered! unless defined?(Rake) # rake preloads all plugins
+SingleCov.covered!
 
 describe SamsonKubernetes do
   describe :stage_permitted_params do
-    it "adds outs" do
+    it "adds ours" do
       Samson::Hooks.fire(:stage_permitted_params).must_include :kubernetes
     end
   end
 
   describe :deploy_permitted_params do
-    it "adds outs" do
+    it "adds ours" do
       params = Samson::Hooks.fire(:deploy_permitted_params).flatten
       params.must_include :kubernetes_rollback
       params.must_include :kubernetes_reuse_build
     end
   end
 
   describe :build_permitted_params do
-    it "adds outs" do
+    it "adds ours" do
       Samson::Hooks.fire(:build_permitted_params).must_include :kubernetes_job
     end
   end
+
+  describe :deploy_group_permitted_params do
+    it "adds ours" do
+      params = Samson::Hooks.fire(:deploy_group_permitted_params).flatten
+      params.must_include cluster_deploy_group_attributes: [:kubernetes_cluster_id, :namespace]
+    end
+  end
 end

plugins/new_relic/test/samson_new_relic/samson_plugin_test.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 require_relative '../test_helper'
 
-SingleCov.covered! unless defined?(Rake) # rake preloads all plugins
+SingleCov.covered!
 
 describe SamsonNewRelic do
   describe :stage_permitted_params do
@@ -23,4 +23,18 @@
       attributes[0].must_equal attributes[1]
     end
   end
+
+  describe ".enabled?" do
+    it "is disabled when KEY was not set" do
+      refute SamsonNewRelic.enabled?
+    end
+
+    describe "when enabled" do
+      with_new_relic_plugin_enabled
+
+      it "is enabled when KEY was set" do
+        assert SamsonNewRelic.enabled?
+      end
+    end
+  end
 end

plugins/pipelines/test/samson_pipelines/samson_plugin_test.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 require_relative '../test_helper'
 
-SingleCov.covered! uncovered: 6 unless defined?(Rake) # rake preloads all plugins
+SingleCov.covered! uncovered: 6
 
 describe SamsonPipelines do
   let(:deploy) { deploys(:succeeded_test) }

plugins/slack_app/test/models/slack_message_test.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 require_relative '../test_helper'
 
-SingleCov.covered! uncovered: 3
+SingleCov.covered!
 
 describe SlackMessage do
   let(:deploy) { deploys(:succeeded_test) }

plugins/slack_app/test/samson_slack_app/samson_plugin_test.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 require_relative '../test_helper'
 
-SingleCov.covered! unless defined?(Rake) # rake preloads all plugins
+SingleCov.covered!
 
 describe SamsonSlackApp do
   let(:deploy) { deploys(:succeeded_test) }

plugins/slack_webhooks/test/samson_slack_webhooks/samson_plugin_test.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 require_relative '../test_helper'
 
-SingleCov.covered! unless defined?(Rake) # rake preloads all plugins
+SingleCov.covered!
 
 describe SamsonSlackWebhooks do
   let(:deploy) { deploys(:succeeded_test) }

plugins/slack_webhooks/test/samson_slack_webhooks/slack_webhooks_service_test.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 require_relative '../test_helper'
 
-SingleCov.covered!
+SingleCov.covered! uncovered: 5
 
 describe SamsonSlackWebhooks::SlackWebhooksService do
   let(:deploy) { deploys(:succeeded_test) }

plugins/zendesk/test/samson_zendesk/samson_plugin_test.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 require_relative '../test_helper'
 
-SingleCov.covered! uncovered: 1 unless defined?(Rake) # rake preloads all plugins
+SingleCov.covered! uncovered: 1
 
 describe SamsonZendesk do
   let(:deploy) { deploys(:succeeded_test) }

test/helpers/application_helper_test.rb

@@ -2,7 +2,7 @@
 # rubocop:disable Metrics/LineLength
 require_relative '../test_helper'
 
-SingleCov.covered! uncovered: 7
+SingleCov.covered! uncovered: 10
 
 describe ApplicationHelper do
   include LocksHelper

test/lib/buddy_check_test.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 require_relative '../test_helper'
 
-SingleCov.covered! uncovered: 6
+SingleCov.covered! uncovered: 4
 
 describe BuddyCheck do
   let(:project) { job.project }

test/lib/samson/env_check_test.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 require_relative '../../test_helper'
 
-SingleCov.covered! unless defined?(Rake) # rake preloads app which loads env check
+SingleCov.covered!
 
 describe Samson::EnvCheck do
   describe '.set?' do

test/lib/samson/hooks_test.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 require_relative '../../test_helper'
 
-SingleCov.covered! uncovered: 13 unless defined?(Rake) # rake preloads all plugins
+SingleCov.covered! uncovered: 14
 
 describe Samson::Hooks do
   let(:number_of_plugins) { Dir['plugins/*'].size }

test/models/changeset/github_user_test.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 require_relative '../../test_helper'
 
-SingleCov.covered! uncovered: 8
+SingleCov.covered! uncovered: 5
 
 describe Changeset::GithubUser do
   describe "#avatar_url" do

test/models/job_test.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 require_relative '../test_helper'
 
-SingleCov.covered! uncovered: 21
+SingleCov.covered! uncovered: 22
 
 describe Job do
   include GitRepoTestHelper

test/test_helper.rb

@@ -87,7 +87,7 @@ def assert_sql_queries(count)
     old = ar_queries
     yield
     new = ar_queries
-    extra = new[old.size..-1].reject { |q| q.include?('information_schema') }
+    extra = new[old.size..-1].reject { |q| q.include?('information_schema') || q.include?('sqlite_temp_master') }
     message = extra.join("\n")
     if count.is_a?(Range)
       assert_includes count, extra.count, message