diff --git a/features/command_line/parallel_test.feature b/features/command_line/parallel_test.feature new file mode 100644 index 0000000000..e3e620ccec --- /dev/null +++ b/features/command_line/parallel_test.feature @@ -0,0 +1,40 @@ +Feature: `--parallel-test` option + + Use the `--parallel-test` option to have RSpec run using multiple threads in parallel + + Scenario: Using `--parallel-test 3` + Given a file named "spec/parallel_test_spec.rb" with: + """ruby + RSpec.configure do |c| + c.before(:suite) { puts "before suite" } + c.after(:suite) { puts "after suite" } + end + + RSpec.describe "parallel run" do + before(:context) { puts "before context" } + before(:example) { puts "before example" } + + it "thread 0 example" do + expect(true).to be(false) + end + + it "thread 1 example" do + expect(true).to be(true) + end + + it "thread 2 example" do + expect(true).to be(true) + end + + after(:example) { puts "after example" } + after(:context) { puts "after context" } + end + """ + When I run `rspec --parallel-test 3` + Then the output should contain "3 examples, 1 failure" + And the output should contain "before suite" + And the output should contain "after suite" + And the output should contain "before context" + And the output should contain "after context" + And the output should contain "before example" + And the output should contain "after example" diff --git a/lib/rspec/core.rb b/lib/rspec/core.rb index 1c5bc7b2f4..67117dfa2c 100644 --- a/lib/rspec/core.rb +++ b/lib/rspec/core.rb @@ -28,6 +28,8 @@ configuration option_parser configuration_options + example_thread_runner + example_group_thread_runner runner example shared_example_group diff --git a/lib/rspec/core/configuration.rb b/lib/rspec/core/configuration.rb index 69d82fe97f..3057cf0493 100644 --- a/lib/rspec/core/configuration.rb +++ b/lib/rspec/core/configuration.rb @@ -267,7 +267,7 @@ def treat_symbols_as_metadata_keys_with_true_values=(value) # @private add_setting :expecting_with_rspec # @private - attr_accessor :filter_manager + attr_accessor :filter_manager, :thread_maximum # @private attr_reader :backtrace_formatter, :ordering_manager @@ -302,6 +302,7 @@ def initialize @requires = [] @libs = [] @derived_metadata_blocks = [] + @thread_maximum = 1 end # @private diff --git a/lib/rspec/core/configuration_options.rb b/lib/rspec/core/configuration_options.rb index 6ebad0fd36..503a4b6932 100644 --- a/lib/rspec/core/configuration_options.rb +++ b/lib/rspec/core/configuration_options.rb @@ -53,7 +53,7 @@ def organize_options UNFORCED_OPTIONS = [ :requires, :profile, :drb, :libs, :files_or_directories_to_run, - :full_description, :full_backtrace, :tty + :full_description, :full_backtrace, :tty, :thread_maximum ].to_set UNPROCESSABLE_OPTIONS = [:formatters].to_set diff --git a/lib/rspec/core/example_group.rb b/lib/rspec/core/example_group.rb index 46e6278dfd..135b1de95c 100644 --- a/lib/rspec/core/example_group.rb +++ b/lib/rspec/core/example_group.rb @@ -469,6 +469,34 @@ def self.run(reporter) end end + # Runs all the examples in this group + def self.run_parallel(reporter, num_threads, mutex, used_threads) + if RSpec.world.wants_to_quit + RSpec.world.clear_remaining_example_groups if top_level? + return + end + reporter.example_group_started(self) + + begin + run_before_context_hooks(new) + example_threads = RSpec::Core::ExampleThreadRunner.new(num_threads, used_threads) + run_examples_parallel(reporter, example_threads, mutex) + ordering_strategy.order(children).map { |child| child.run_parallel(reporter, num_threads, mutex, used_threads) } + # ensure all examples in the group are done before running 'after :all' + # this does NOT prevent utilization of other available threads + example_threads.wait_for_completion + rescue Pending::SkipDeclaredInExample => ex + for_filtered_examples(reporter) { |example| example.skip_with_exception(reporter, ex) } + rescue Exception => ex + RSpec.world.wants_to_quit = true if fail_fast? + for_filtered_examples(reporter) { |example| example.fail_with_exception(reporter, ex) } + ensure + run_after_context_hooks(new) + before_context_ivars.clear + reporter.example_group_finished(self) + end + end + # @private def self.ordering_strategy order = metadata.fetch(:order, :global) @@ -497,6 +525,18 @@ def self.run_examples(reporter) end.all? end + # @private + def self.run_examples_parallel(reporter, threads, mutex) + ordering_strategy.order(filtered_examples).map do |example| + next if RSpec.world.wants_to_quit + instance = new + set_ivars(instance, before_context_ivars) + mutex.synchronize do + threads.run(example, instance, reporter) + end + end + end + # @private def self.for_filtered_examples(reporter, &block) filtered_examples.each(&block) diff --git a/lib/rspec/core/example_group_thread_runner.rb b/lib/rspec/core/example_group_thread_runner.rb new file mode 100644 index 0000000000..2ee8240f94 --- /dev/null +++ b/lib/rspec/core/example_group_thread_runner.rb @@ -0,0 +1,49 @@ +module RSpec + module Core + # ExampleGroupThreadRunner is a class used to execute [ExampleGroup] + # classes in parallel as part of rspec-core. When running in parallel + # the order of example groups will not be honoured. + # This class is used to ensure that we have a way of keeping track of + # the number of threads being created and preventing utilization of + # more than the specified number + # Additionally, this class will contain a mutex used to prevent access + # to shared variables within sub-threads + class ExampleGroupThreadRunner + attr_accessor :thread_array, :max_threads, :mutex, :used_threads + + # Creates a new instance of ExampleGroupThreadRunner. + # @param max_threads [Integer] the maximum limit of threads that can be used + # @param mutex [Mutex] a semaphore used to prevent access to shared variables in + # sub-threads such as those used by [ExampleThreadRunner] + # @param used_threads [Integer] the current number of threads being used + def initialize(max_threads = 1, mutex = Mutex.new, used_threads = 0) + @max_threads = max_threads + @mutex = mutex + @used_threads = used_threads + @thread_array = [] + end + + # Method will run an [ExampleGroup] inside a [Thread] to prevent blocking + # execution. The new [Thread] is added to an array for tracking and + # will automatically remove itself when done + # @param example_group [ExampleGroup] the group to be run inside a [Thread] + # @param reporter [Reporter] the passed in reporting class used for + # tracking + def run(example_group, reporter) + @thread_array.push Thread.start { + example_group.run_parallel(reporter, @max_threads, @mutex, @used_threads) + @thread_array.delete Thread.current + } + end + + # Method will wait for all threads to complete. On completion threads + # remove themselves from the @thread_array so an empty array means they + # completed + def wait_for_completion + @thread_array.each do |t| + t.join + end + end + end + end +end diff --git a/lib/rspec/core/example_thread_runner.rb b/lib/rspec/core/example_thread_runner.rb new file mode 100644 index 0000000000..7f05c33f28 --- /dev/null +++ b/lib/rspec/core/example_thread_runner.rb @@ -0,0 +1,55 @@ +module RSpec + module Core + # ExampleThreadRunner is a class used to execute [Example] classes in + # parallel as part of rspec-core. When running in parallel the order + # of examples will not be honoured. + # This class is used to ensure that we have a way of keeping track of + # the number of threads being created and preventing utilization of + # more than the specified number + class ExampleThreadRunner + attr_accessor :num_threads, :thread_array, :used_threads + + # Creates a new instance of ExampleThreadRunner. + # @param num_threads [Integer] the maximum limit of threads that can be used + # @param used_threads [Integer] the current number of threads being used + def initialize(num_threads, used_threads) + @num_threads = num_threads + @thread_array = [] + @used_threads = used_threads + end + + # Method will check global utilization of threads and if that number is + # at or over the allocated maximum it will wait until a thread is available + def wait_for_available_thread + while @used_threads.to_i >= @num_threads.to_i + sleep 0.1 + end + end + + # Method will run the specified example within an available thread or + # will wait for a thread to become available if none currently are + # @param example [Example] the example to be executed in a [Thread] + # @param instance the instance of an ExampleGroup subclass + # @param reporter [Reporter] the passed in reporting class used for + # tracking + def run(example, instance, reporter) + wait_for_available_thread + @thread_array.push Thread.start { + example.run(instance, reporter) + @thread_array.delete Thread.current # remove from local scope + @used_threads -= 1 + } + @used_threads += 1 + end + + # Method will wait for all threads to complete. On completion threads + # remove themselves from the @thread_array so an empty array means they + # completed + def wait_for_completion + @thread_array.each do |t| + t.join + end + end + end + end +end diff --git a/lib/rspec/core/option_parser.rb b/lib/rspec/core/option_parser.rb index 03e91fefb0..c8da924bdc 100644 --- a/lib/rspec/core/option_parser.rb +++ b/lib/rspec/core/option_parser.rb @@ -137,6 +137,20 @@ def parser(options) $VERBOSE = true end + parser.on('--parallel-test NUMBER', 'Run the tests with the specified number of parallel threads (default: 1).') do |n| + options[:thread_maximum] = if !n.nil? + begin + Integer(n) + rescue ArgumentError + RSpec.warning "Non integer specified as number of parallel threads, seperate " + + "your path from options with a space e.g. " + + "`rspec --parallel-test #{n}`", + :call_site => nil + 1 + end + end + end + parser.separator <<-FILTERING **** Filtering/tags **** diff --git a/lib/rspec/core/runner.rb b/lib/rspec/core/runner.rb index 5c452fd57d..0aef9b49ca 100644 --- a/lib/rspec/core/runner.rb +++ b/lib/rspec/core/runner.rb @@ -83,7 +83,12 @@ def initialize(options, configuration=RSpec.configuration, world=RSpec.world) # @param out [IO] output stream def run(err, out) setup(err, out) - run_specs(@world.ordered_example_groups) + if @options.options[:thread_maximum].nil? + run_specs(@world.ordered_example_groups) + else + require 'thread' + run_specs_parallel(@world.ordered_example_groups) + end end # Wires together the various configuration objects and state holders. @@ -116,6 +121,33 @@ def run_specs(example_groups) end end + # Runs the provided example groups in parallel. + # + # @param example_groups [Array] groups to run + # @return [Fixnum] exit status code. 0 if all specs passed, + # or the configured failure exit code (1 by default) if specs + # failed. + def run_specs_parallel(example_groups) + @configuration.reporter.report(@world.example_count(example_groups)) do |reporter| + begin + hook_context = SuiteHookContext.new + @configuration.hooks.run(:before, :suite, hook_context) + + group_threads = RSpec::Core::ExampleGroupThreadRunner.new(@configuration.thread_maximum) + example_groups.each { |g| group_threads.run(g, reporter) } + group_threads.wait_for_completion + + example_groups.all? do |g| + result_for_this_group = g.filtered_examples.all? { |example| example.metadata[:execution_result].exception.nil? } + results_for_descendants = g.children.all? { |child| child.filtered_examples.all? { |example| example.metadata[:execution_result].exception.nil? } } + result_for_this_group && results_for_descendants + end ? 0 : @configuration.failure_exit_code + ensure + @configuration.hooks.run(:after, :suite, hook_context) + end + end + end + # @private def self.disable_autorun! @autorun_disabled = true diff --git a/spec/command_line/parallel_spec.rb b/spec/command_line/parallel_spec.rb new file mode 100644 index 0000000000..a79ba893ec --- /dev/null +++ b/spec/command_line/parallel_spec.rb @@ -0,0 +1,83 @@ +require 'spec_helper' + +RSpec.describe 'command line', :ui, :slow do + before :each do + @out = StringIO.new + write_file 'spec/parallel_spec.rb', """ + RSpec.describe 'parallel run' do + it 'thread 0 example' do + sleep 1 + fail + end + + it 'thread 1 example' do + sleep 1 + pass + end + + it 'thread 2 example' do + sleep 1 + pass + end + + it 'thread 3 example' do + sleep 1 + fail + end + + it 'thread 4 example' do + sleep 1 + pass + end + + it 'thread 5 example' do + sleep 1 + pass + end + end + """ + end + + describe '--parallel-test' do + it '1 thread' do + run_command 'spec/parallel_spec.rb --parallel-test 1' + output_str = @out.string + validate_output(output_str) + seconds = get_seconds(output_str).to_i + expect(seconds).to be >= 6 + end + + it '3 threads' do + run_command 'spec/parallel_spec.rb --parallel-test 3' + output_str = @out.string + validate_output(output_str) + seconds = get_seconds(output_str).to_i + expect(seconds).to be >= 2 + expect(seconds).to be < 7 + end + + it '6 threads' do + run_command 'spec/parallel_spec.rb --parallel-test 6' + output_str = @out.string + validate_output(output_str) + seconds = get_seconds(output_str).to_i + expect(seconds).to be >= 0 + expect(seconds).to be < 4 + end + end + + def run_command(cmd) + in_current_dir do + RSpec::Core::Runner.run(cmd.split, @out, @out) + end + end + + def validate_output(output_str) + expect(output_str).to include("6 examples, 2 failures"), "output_str: #{output_str}" + expect(output_str).to include("Finished in "), "output_str: #{output_str}" + end + + def get_seconds(output_str) + return output_str[/Finished in (.*) second/, 1] + end +end