From 26821d9b572815a39c8ecb2e19375b2abff68730 Mon Sep 17 00:00:00 2001 From: eileencodes Date: Wed, 20 Dec 2017 16:59:41 -0500 Subject: [PATCH] Add test parallelization to Rails Provides both a forked process and threaded parallelization options. To use add `parallelize` to your test suite. Takes a `workers` argument that controls how many times the process is forked. For each process a new database will be created suffixed with the worker number; test-database-0 and test-database-1 respectively. If `ENV["PARALLEL_WORKERS"]` is set the workers argument will be ignored and the environment variable will be used instead. This is useful for CI environments, or other environments where you may need more workers than you do for local testing. If the number of workers is set to `1` or fewer, the tests will not be parallelized. The default parallelization method is to fork processes. If you'd like to use threads instead you can pass `with: :threads` to the `parallelize` method. Note the threaded parallelization does not create multiple database and will not work with system tests at this time. parallelize(workers: 2, with: :threads) The threaded parallelization uses Minitest's parallel exector directly. The processes paralleliztion uses a Ruby Drb server. For parallelization via threads a setup hook and cleanup hook are provided. ``` class ActiveSupport::TestCase parallelize_setup do |worker| # setup databases end parallelize_teardown do |worker| # cleanup database end parallelize(workers: 2) end ``` [Eileen M. Uchitelle, Aaron Patterson] --- activerecord/lib/active_record.rb | 1 + activerecord/lib/active_record/fixtures.rb | 5 +- .../lib/active_record/test_databases.rb | 38 +++++++ activesupport/CHANGELOG.md | 5 + activesupport/lib/active_support/test_case.rb | 86 +++++++++++++++ .../active_support/testing/parallelization.rb | 102 ++++++++++++++++++ guides/source/testing.md | 83 ++++++++++++++ .../app/templates/test/test_helper.rb.tt | 7 ++ railties/lib/rails/test_help.rb | 1 + railties/test/application/test_runner_test.rb | 82 +++++++++++++- railties/test/application/test_test.rb | 5 + railties/test/isolation/abstract_unit.rb | 7 +- 12 files changed, 415 insertions(+), 7 deletions(-) create mode 100644 activerecord/lib/active_record/test_databases.rb create mode 100644 activesupport/lib/active_support/testing/parallelization.rb diff --git a/activerecord/lib/active_record.rb b/activerecord/lib/active_record.rb index b4377ad6beb0e..d43378c64f3c9 100644 --- a/activerecord/lib/active_record.rb +++ b/activerecord/lib/active_record.rb @@ -163,6 +163,7 @@ module Tasks "active_record/tasks/postgresql_database_tasks" end + autoload :TestDatabases, "active_record/test_databases" autoload :TestFixtures, "active_record/fixtures" def self.eager_load! diff --git a/activerecord/lib/active_record/fixtures.rb b/activerecord/lib/active_record/fixtures.rb index d9a75d9ad675d..8f022ff7a7ae2 100644 --- a/activerecord/lib/active_record/fixtures.rb +++ b/activerecord/lib/active_record/fixtures.rb @@ -874,6 +874,7 @@ def after_teardown # :nodoc: class_attribute :use_instantiated_fixtures, default: false # true, false, or :no_instances class_attribute :pre_loaded_fixtures, default: false class_attribute :config, default: ActiveRecord::Base + class_attribute :lock_threads, default: true end module ClassMethods @@ -973,7 +974,7 @@ def setup_fixtures(config = ActiveRecord::Base) @fixture_connections = enlist_fixture_connections @fixture_connections.each do |connection| connection.begin_transaction joinable: false - connection.pool.lock_thread = true + connection.pool.lock_thread = true if lock_threads end # When connections are established in the future, begin a transaction too @@ -989,7 +990,7 @@ def setup_fixtures(config = ActiveRecord::Base) if connection && !@fixture_connections.include?(connection) connection.begin_transaction joinable: false - connection.pool.lock_thread = true + connection.pool.lock_thread = true if lock_threads @fixture_connections << connection end end diff --git a/activerecord/lib/active_record/test_databases.rb b/activerecord/lib/active_record/test_databases.rb new file mode 100644 index 0000000000000..606a3b0fb5060 --- /dev/null +++ b/activerecord/lib/active_record/test_databases.rb @@ -0,0 +1,38 @@ +# frozen_string_literal: true + +require "active_support/testing/parallelization" + +module ActiveRecord + module TestDatabases # :nodoc: + ActiveSupport::Testing::Parallelization.after_fork_hook do |i| + create_and_migrate(i, spec_name: Rails.env) + end + + ActiveSupport::Testing::Parallelization.run_cleanup_hook do |i| + drop(i, spec_name: Rails.env) + end + + def self.create_and_migrate(i, spec_name:) + old, ENV["VERBOSE"] = ENV["VERBOSE"], "false" + + connection_spec = ActiveRecord::Base.configurations[spec_name] + + connection_spec["database"] += "-#{i}" + ActiveRecord::Tasks::DatabaseTasks.create(connection_spec) + ActiveRecord::Base.establish_connection(connection_spec) + ActiveRecord::Tasks::DatabaseTasks.migrate + ensure + ActiveRecord::Base.establish_connection(ActiveRecord::Base.configurations[Rails.env]) + ENV["VERBOSE"] = old + end + + def self.drop(i, spec_name:) + old, ENV["VERBOSE"] = ENV["VERBOSE"], "false" + connection_spec = ActiveRecord::Base.configurations[spec_name] + + ActiveRecord::Tasks::DatabaseTasks.drop(connection_spec) + ensure + ENV["VERBOSE"] = old + end + end +end diff --git a/activesupport/CHANGELOG.md b/activesupport/CHANGELOG.md index fc47c76e99f69..2976040a59c7e 100644 --- a/activesupport/CHANGELOG.md +++ b/activesupport/CHANGELOG.md @@ -1,2 +1,7 @@ +* Adds parallel testing to Rails + + Parallelize your test suite with forked processes or threads. + + *Eileen M. Uchitelle*, *Aaron Patterson* Please check [5-2-stable](https://github.com/rails/rails/blob/5-2-stable/activesupport/CHANGELOG.md) for previous changes. diff --git a/activesupport/lib/active_support/test_case.rb b/activesupport/lib/active_support/test_case.rb index d1f7e6ea095b1..3588a5ac02101 100644 --- a/activesupport/lib/active_support/test_case.rb +++ b/activesupport/lib/active_support/test_case.rb @@ -11,6 +11,7 @@ require "active_support/testing/constant_lookup" require "active_support/testing/time_helpers" require "active_support/testing/file_fixtures" +require "active_support/testing/parallelization" module ActiveSupport class TestCase < ::Minitest::Test @@ -39,6 +40,91 @@ def test_order=(new_order) def test_order ActiveSupport.test_order ||= :random end + + # Parallelizes the test suite. + # + # Takes a `workers` argument that controls how many times the process + # is forked. For each process a new database will be created suffixed + # with the worker number. + # + # test-database-0 + # test-database-1 + # + # If `ENV["PARALLEL_WORKERS"]` is set the workers argument will be ignored + # and the environment variable will be used instead. This is useful for CI + # environments, or other environments where you may need more workers than + # you do for local testing. + # + # If the number of workers is set to `1` or fewer, the tests will not be + # parallelized. + # + # The default parallelization method is to fork processes. If you'd like to + # use threads instead you can pass `with: :threads` to the `parallelize` + # method. Note the threaded parallelization does not create multiple + # database and will not work with system tests at this time. + # + # parallelize(workers: 2, with: :threads) + # + # The threaded parallelization uses Minitest's parallel exector directly. + # The processes paralleliztion uses a Ruby Drb server. + def parallelize(workers: 2, with: :processes) + workers = ENV["PARALLEL_WORKERS"].to_i if ENV["PARALLEL_WORKERS"] + + return if workers <= 1 + + executor = case with + when :processes + Testing::Parallelization.new(workers) + when :threads + Minitest::Parallel::Executor.new(workers) + else + raise ArgumentError, "#{with} is not a supported parallelization exectutor." + end + + self.lock_threads = false if defined?(self.lock_threads) && with == :threads + + Minitest.parallel_executor = executor + + parallelize_me! + end + + # Set up hook for parallel testing. This can be used if you have multiple + # databases or any behavior that needs to be run after the process is forked + # but before the tests run. + # + # Note: this feature is not available with the threaded parallelization. + # + # In your +test_helper.rb+ add the following: + # + # class ActiveSupport::TestCase + # parallelize_setup do + # # create databases + # end + # end + def parallelize_setup(&block) + ActiveSupport::Testing::Parallelization.after_fork_hook do |worker| + yield worker + end + end + + # Clean up hook for parallel testing. This can be used to drop databases + # if your app uses multiple write/read databases or other clean up before + # the tests finish. This runs before the forked process is closed. + # + # Note: this feature is not available with the threaded parallelization. + # + # In your +test_helper.rb+ add the following: + # + # class ActiveSupport::TestCase + # parallelize_teardown do + # # drop databases + # end + # end + def parallelize_teardown(&block) + ActiveSupport::Testing::Parallelization.run_cleanup_hook do |worker| + yield worker + end + end end alias_method :method_name, :name diff --git a/activesupport/lib/active_support/testing/parallelization.rb b/activesupport/lib/active_support/testing/parallelization.rb new file mode 100644 index 0000000000000..59c8486f41c05 --- /dev/null +++ b/activesupport/lib/active_support/testing/parallelization.rb @@ -0,0 +1,102 @@ +# frozen_string_literal: true + +require "drb" +require "drb/unix" + +module ActiveSupport + module Testing + class Parallelization # :nodoc: + class Server + include DRb::DRbUndumped + + def initialize + @queue = Queue.new + end + + def record(reporter, result) + reporter.synchronize do + reporter.record(result) + end + end + + def <<(o) + @queue << o + end + + def pop; @queue.pop; end + end + + @after_fork_hooks = [] + + def self.after_fork_hook(&blk) + @after_fork_hooks << blk + end + + def self.after_fork_hooks + @after_fork_hooks + end + + @run_cleanup_hooks = [] + + def self.run_cleanup_hook(&blk) + @run_cleanup_hooks << blk + end + + def self.run_cleanup_hooks + @run_cleanup_hooks + end + + def initialize(queue_size) + @queue_size = queue_size + @queue = Server.new + @pool = [] + + @url = DRb.start_service("drbunix:", @queue).uri + end + + def after_fork(worker) + self.class.after_fork_hooks.each do |cb| + cb.call(worker) + end + end + + def run_cleanup(worker) + self.class.run_cleanup_hooks.each do |cb| + cb.call(worker) + end + end + + def start + @pool = @queue_size.times.map do |worker| + fork do + DRb.stop_service + + after_fork(worker) + + queue = DRbObject.new_with_uri(@url) + + while job = queue.pop + klass = job[0] + method = job[1] + reporter = job[2] + result = Minitest.run_one_method(klass, method) + + queue.record(reporter, result) + end + + run_cleanup(worker) + end + end + end + + def <<(work) + @queue << work + end + + def shutdown + @queue_size.times { @queue << nil } + @pool.each { |pid| Process.waitpid pid } + end + end + end +end diff --git a/guides/source/testing.md b/guides/source/testing.md index af6127f4a5d6d..5d2b8bc5d65e7 100644 --- a/guides/source/testing.md +++ b/guides/source/testing.md @@ -462,6 +462,89 @@ Rails options: -c, --[no-]color Enable color in the output ``` +Parallel Testing +---------------- + +Parallel testing allows you to parallelize your test suite. While forking processes is the +default method, threading is supported as well. Running tests in parallel reduces the time it +takes your entire test suite to run. + +## Parallel testing with processes + +The default parallelization method is to fork processes using Ruby's DRb system. The processes +are forked based on the number of workers provided. The default is 2, but can be changed by the +number passed to the parallelize method. Active Record automatically handles creating and +migrating a new database for each worker to use. + +To enable parallelization add the following to your `test_helper.rb`: + +``` +class ActiveSupport::TestCase + parallelize(workers: 2) +end +``` + +The number of workers passed is the number of times the process will be forked. You may want to +parallelize your local test suite differently from your CI, so an environment variable is provided +to be able to easily change the number of workers a test run should use: + +``` +PARALLEL_WORKERS=15 bin/rails test +``` + +When parallelizing tests, Active Record automatically handles creating and migrating a database for each +process. The databases will be suffixed with the number corresponding to the worker. For example, if you +have 2 workers the tests will create `test-database-0` and `test-database-1` respectively. + +If the number of workers passed is 1 or fewer the processes will not be forked and the tests will not +be parallelized and the tests will use the original `test-database` database. + +Two hooks are provided, one runs when the process is forked, and one runs before the processes are closed. +These can be useful if your app uses multiple databases or perform other tasks that depend on the number of +workers. + +The `parallelize_setup` method is called right after the processes are forked. The `parallelize_teardown` metod +is called right before the processes are closed. + +``` +class ActiveSupport::TestCase + parallelize_setup do |worker| + # setup databases + end + + parallelize_teardown do |worker| + # cleanup database + end + + parallelize(workers: 2) +end +``` + +These methods are not needed or available when using parallel testing with threads. + +## Parallel testing with threads + +If you prefer using threads or are using JRuby, a threaded parallelization option is provided. The threaded +parallelizer is backed by Minitest's `Parallel::Executor`. + +To change the parallelization method to use threads over forks put the following in your `test_helper.rb` + +``` +class ActiveSupport::TestCase + parallelize(workers: 2, with: :threads) +end +``` + +Rails applications generated from JRuby will automatically include the `with: :threads` option. + +The number of workers passed to `parallelize` determines the number of threads the tests will use. You may +want to parallelize your local test suite differently from your CI, so an environment variable is provided +to be able to easily change the number of workers a test run should use: + +``` +PARALLEL_WORKERS=15 bin/rails test +``` + The Test Database ----------------- diff --git a/railties/lib/rails/generators/rails/app/templates/test/test_helper.rb.tt b/railties/lib/rails/generators/rails/app/templates/test/test_helper.rb.tt index 52d68cc77ce2f..c918b57ecaab8 100644 --- a/railties/lib/rails/generators/rails/app/templates/test/test_helper.rb.tt +++ b/railties/lib/rails/generators/rails/app/templates/test/test_helper.rb.tt @@ -3,6 +3,13 @@ require_relative '../config/environment' require 'rails/test_help' class ActiveSupport::TestCase + # Run tests in parallel with specified workers +<% if defined?(JRUBY_VERSION) -%> + parallelize(workers: 2, with: :threads) +<%- else -%> + parallelize(workers: 2) +<% end -%> + <% unless options[:skip_active_record] -%> # Setup all fixtures in test/fixtures/*.yml for all tests in alphabetical order. fixtures :all diff --git a/railties/lib/rails/test_help.rb b/railties/lib/rails/test_help.rb index 76c28ac85ef59..4bd7d74b04457 100644 --- a/railties/lib/rails/test_help.rb +++ b/railties/lib/rails/test_help.rb @@ -22,6 +22,7 @@ module ActiveSupport class TestCase + include ActiveRecord::TestDatabases include ActiveRecord::TestFixtures self.fixture_path = "#{Rails.root}/test/fixtures/" self.file_fixture_path = fixture_path + "files" diff --git a/railties/test/application/test_runner_test.rb b/railties/test/application/test_runner_test.rb index a01325fdb89e9..fe1dd59990ba5 100644 --- a/railties/test/application/test_runner_test.rb +++ b/railties/test/application/test_runner_test.rb @@ -502,10 +502,10 @@ def test_run_app_without_rails_loaded end def test_output_inline_by_default - create_test_file :models, "post", pass: false + create_test_file :models, "post", pass: false, print: false output = run_test_command("test/models/post_test.rb") - expect = %r{Running:\n\nPostTest\nF\n\nFailure:\nPostTest#test_truth \[[^\]]+test/models/post_test.rb:6\]:\nwups!\n\nbin/rails test test/models/post_test.rb:4\n\n\n\n} + expect = %r{Running:\n\nF\n\nFailure:\nPostTest#test_truth \[[^\]]+test/models/post_test.rb:6\]:\nwups!\n\nbin/rails test test/models/post_test.rb:4\n\n\n\n} assert_match expect, output end @@ -523,6 +523,29 @@ def test_fail_fast capture(:stderr) { run_test_command("test/models/post_test.rb --fail-fast", stderr: true) }) end + def test_run_in_parallel_with_processes + file_name = create_parallel_processes_test_file + + output = run_test_command(file_name) + + assert_match %r{Finished in.*\n2 runs, 2 assertions}, output + end + + def test_run_in_parallel_with_threads + app_path("/test/test_helper.rb") do |file_name| + file = File.read(file_name) + file.sub!(/parallelize\(([^\)]*)\)/, "parallelize(\\1, with: :threads)") + puts file + File.write(file_name, file) + end + + file_name = create_parallel_threads_test_file + + output = run_test_command(file_name) + + assert_match %r{Finished in.*\n2 runs, 2 assertions}, output + end + def test_raise_error_when_specified_file_does_not_exist error = capture(:stderr) { run_test_command("test/not_exists.rb", stderr: true) } assert_match(%r{cannot load such file.+test/not_exists\.rb}, error) @@ -800,19 +823,70 @@ def test_environment RUBY end - def create_test_file(path = :unit, name = "test", pass: true) + def create_test_file(path = :unit, name = "test", pass: true, print: true) app_file "test/#{path}/#{name}_test.rb", <<-RUBY require 'test_helper' class #{name.camelize}Test < ActiveSupport::TestCase def test_truth - puts "#{name.camelize}Test" + puts "#{name.camelize}Test" if #{print} assert #{pass}, 'wups!' end end RUBY end + def create_parallel_processes_test_file + app_file "test/models/parallel_test.rb", <<-RUBY + require 'test_helper' + + class ParallelTest < ActiveSupport::TestCase + RD1, WR1 = IO.pipe + RD2, WR2 = IO.pipe + + test "one" do + WR1.close + assert_equal "x", RD1.read(1) # blocks until two runs + + RD2.close + WR2.write "y" # Allow two to run + WR2.close + end + + test "two" do + RD1.close + WR1.write "x" # Allow one to run + WR1.close + + WR2.close + assert_equal "y", RD2.read(1) # blocks until one runs + end + end + RUBY + end + + def create_parallel_threads_test_file + app_file "test/models/parallel_test.rb", <<-RUBY + require 'test_helper' + + class ParallelTest < ActiveSupport::TestCase + Q1 = Queue.new + Q2 = Queue.new + test "one" do + assert_equal "x", Q1.pop # blocks until two runs + + Q2 << "y" + end + + test "two" do + Q1 << "x" + + assert_equal "y", Q2.pop # blocks until one runs + end + end + RUBY + end + def create_env_test app_file "test/unit/env_test.rb", <<-RUBY require 'test_helper' diff --git a/railties/test/application/test_test.rb b/railties/test/application/test_test.rb index 0a51e986565bd..fb43bebfbec14 100644 --- a/railties/test/application/test_test.rb +++ b/railties/test/application/test_test.rb @@ -7,10 +7,15 @@ class TestTest < ActiveSupport::TestCase include ActiveSupport::Testing::Isolation def setup + @old = ENV["PARALLEL_WORKERS"] + ENV["PARALLEL_WORKERS"] = "0" + build_app end def teardown + ENV["PARALLEL_WORKERS"] = @old + teardown_app end diff --git a/railties/test/isolation/abstract_unit.rb b/railties/test/isolation/abstract_unit.rb index 6568a356d647b..0a4d2a916757b 100644 --- a/railties/test/isolation/abstract_unit.rb +++ b/railties/test/isolation/abstract_unit.rb @@ -38,7 +38,12 @@ def tmp_path(*args) end def app_path(*args) - tmp_path(*%w[app] + args) + path = tmp_path(*%w[app] + args) + if block_given? + yield path + else + path + end end def framework_path