From b4959a96db04043bd0f60506c365b7e647a1e285 Mon Sep 17 00:00:00 2001 From: bicarbon8 Date: Tue, 13 May 2014 11:46:59 +0100 Subject: [PATCH 01/13] These changes add handling of an additional parameter "--parallel-test" that takes a number as input. This number will then be used to limit execution of examples to the specified number of parallel threads. To accomplish this the ExampleGroups each are started in a sub-thread which can then start up to the specified number of example threads running in parallel. A global maximum is used to prevent going over the specified number and execution of additional threads will wait until an available thread is free for cases where the maximum number of parallel threads are in use. Additionally, these changes ensure that Before / After Suite and Before / After All calls are executed only once per expected grouping, but that Before / After Each calls are executed before and after each example as part of the parallel thread. The advantage of this change is that it fully supports all platforms supported by rspec (unlike parallel_tests gem which does not work correctly on all Windows systems) and the output remains serially reported so you avoid the need to stitch together test reports at the completion of testing (as can be required with solutions like parallel_tests and prspec gems where parallel execution is actually split between multiple instances of rspec). --- lib/rspec/core.rb | 4 ++ lib/rspec/core/configuration_options.rb | 4 +- lib/rspec/core/example.rb | 11 +++ lib/rspec/core/example_group.rb | 27 ++++--- lib/rspec/core/example_group_thread_runner.rb | 31 ++++++++ lib/rspec/core/example_thread_runner.rb | 72 +++++++++++++++++++ lib/rspec/core/option_parser.rb | 14 ++++ lib/rspec/core/runner.rb | 37 +++++++--- 8 files changed, 181 insertions(+), 19 deletions(-) create mode 100644 lib/rspec/core/example_group_thread_runner.rb create mode 100644 lib/rspec/core/example_thread_runner.rb diff --git a/lib/rspec/core.rb b/lib/rspec/core.rb index 1c5bc7b2f4..c45aabca84 100644 --- a/lib/rspec/core.rb +++ b/lib/rspec/core.rb @@ -1,4 +1,6 @@ $_rspec_core_load_started_at = Time.now +require 'thread' + require 'rbconfig' require "rspec/support" @@ -28,6 +30,8 @@ configuration option_parser configuration_options + example_thread_runner + example_group_thread_runner runner example shared_example_group diff --git a/lib/rspec/core/configuration_options.rb b/lib/rspec/core/configuration_options.rb index 6ebad0fd36..d0b114fa3c 100644 --- a/lib/rspec/core/configuration_options.rb +++ b/lib/rspec/core/configuration_options.rb @@ -53,10 +53,10 @@ def organize_options UNFORCED_OPTIONS = [ :requires, :profile, :drb, :libs, :files_or_directories_to_run, - :full_description, :full_backtrace, :tty + :full_description, :full_backtrace, :tty, :parallel_test ].to_set - UNPROCESSABLE_OPTIONS = [:formatters].to_set + UNPROCESSABLE_OPTIONS = [:formatters, :parallel_test].to_set def force?(key) !UNFORCED_OPTIONS.include?(key) diff --git a/lib/rspec/core/example.rb b/lib/rspec/core/example.rb index 97725de8c8..0a16c6fa16 100644 --- a/lib/rspec/core/example.rb +++ b/lib/rspec/core/example.rb @@ -79,6 +79,16 @@ def description RSpec.configuration.format_docstrings_block.call(description) end + # @attr_accessor + # + # Holds the completion status of the example (nil if not completed) + attr_accessor :succeeded + + # Convenience method for getting success status of example + def succeeded? + @succeeded + end + # @attr_reader # # Returns the first exception raised in the context of running this @@ -146,6 +156,7 @@ def run(example_group_instance, reporter) begin run_before_example @example_group_instance.instance_exec(self, &@example_block) + @succeeded = true # this will not be set to true if a failure occurs if pending? Pending.mark_fixed! self diff --git a/lib/rspec/core/example_group.rb b/lib/rspec/core/example_group.rb index 46e6278dfd..d08ab0c881 100644 --- a/lib/rspec/core/example_group.rb +++ b/lib/rspec/core/example_group.rb @@ -445,7 +445,7 @@ def self.run_after_context_hooks(example_group_instance) end # Runs all the examples in this group - def self.run(reporter) + def self.run(reporter, num_threads=1) if RSpec.world.wants_to_quit RSpec.world.clear_remaining_example_groups if top_level? return @@ -454,9 +454,12 @@ def self.run(reporter) begin run_before_context_hooks(new) - result_for_this_group = run_examples(reporter) - results_for_descendants = ordering_strategy.order(children).map { |child| child.run(reporter) }.all? - result_for_this_group && results_for_descendants + example_threads = RSpec::Core::ExampleThreadRunner.new(num_threads) + run_examples(reporter, example_threads) + ordering_strategy.order(children).map {|child| child.run(reporter, num_threads)} + # ensure all examples in the group are done before running 'after :all' + # this does NOT prevent utilization of other available threads + example_threads.wait_for_completion rescue Pending::SkipDeclaredInExample => ex for_filtered_examples(reporter) {|example| example.skip_with_exception(reporter, ex) } rescue Exception => ex @@ -469,6 +472,14 @@ def self.run(reporter) end end + # Method will report the success / failure status of this ExampleGroup + # based on the success / failure status of its contained Examples + def self.succeeded? + filtered_examples.map do |example| + example.succeeded? + end.all? + end + # @private def self.ordering_strategy order = metadata.fetch(:order, :global) @@ -486,15 +497,13 @@ def self.ordering_strategy end # @private - def self.run_examples(reporter) + def self.run_examples(reporter, threads) ordering_strategy.order(filtered_examples).map do |example| next if RSpec.world.wants_to_quit instance = new set_ivars(instance, before_context_ivars) - succeeded = example.run(instance, reporter) - RSpec.world.wants_to_quit = true if fail_fast? && !succeeded - succeeded - end.all? + threads.run(example, instance, reporter) + end end # @private diff --git a/lib/rspec/core/example_group_thread_runner.rb b/lib/rspec/core/example_group_thread_runner.rb new file mode 100644 index 0000000000..8c91632863 --- /dev/null +++ b/lib/rspec/core/example_group_thread_runner.rb @@ -0,0 +1,31 @@ +module RSpec + module Core + class ExampleGroupThreadRunner + attr_accessor :thread_array + + def initialize + @thread_array = [] + end + + def run(examplegroup, reporter, num_threads=1) + @thread_array.push Thread.start { + # puts "Starting examplegroup '#{examplegroup.description}'..." + examplegroup.run(reporter, num_threads) + # puts "Examplegroup '#{examplegroup.description}' completed." + @thread_array.delete Thread.current # remove from local scope + } + end + + # Method will wait for all threads to complete. On completion threads + # remove themselves from the @thread_array so an empty array means they + # completed + def wait_for_completion + # wait for threads to complete + while @thread_array.length > 0 + # puts "Waiting for #{@thread_array.length} group threads to complete." + sleep 1 #0.1 + end + end + end + end +end \ No newline at end of file diff --git a/lib/rspec/core/example_thread_runner.rb b/lib/rspec/core/example_thread_runner.rb new file mode 100644 index 0000000000..1384bce0f6 --- /dev/null +++ b/lib/rspec/core/example_thread_runner.rb @@ -0,0 +1,72 @@ +module RSpec + module Core + class ExampleThreadRunner + attr_accessor :num_threads, :thread_array, :fname, :lock + + def initialize(num_threads) + @num_threads = num_threads # used to track the local usage of threads + # puts "Creating ExampleThreadRunner object with #{@num_threads} threads." + @thread_array = [] + $used_threads = $used_threads || 0 # used to track the global usage of threads + @fname = '.examplelock' + if !File.exists? @fname + File.new(@fname, "a+") { |f| f.write "lock" } + end + end + + # Method will check global utilization of threads and if that number is + # at or over the allocated maximum it will wait until a thread is available + def wait_for_available_thread + # puts "Global threads in use = #{$used_threads}." + # puts "Local threads in use = #{@thread_array.length}." + # wait for available thread if we've reached our global limit + while $used_threads.to_i >= @num_threads.to_i + # puts "Waiting for available thread. Running = #{$used_threads}; Max = #{@num_threads}" + sleep 1 #0.1 + end + end + + # Method will run the specified example within an available thread or + # will wait for a thread to become available if none currently are + def run(example, instance, reporter) + # puts "Setting lock for example '#{example.description}'..." + set_lock + wait_for_available_thread + @thread_array.push Thread.start { + # puts "Starting example '#{example.description}'..." + example.run(instance, reporter) + # puts "Example '#{example.description}' completed." + @thread_array.delete Thread.current # remove from local scope + $used_threads -= 1 # remove from global scope + } + $used_threads += 1 # add to global scope + ensure + # puts "Releasing lock for example '#{example.description}'..." + release_lock + # puts "Lock for '#{example.description}' released." + end + + # Method will wait for all threads to complete. On completion threads + # remove themselves from the @thread_array so an empty array means they + # completed + def wait_for_completion + # wait for threads to complete + while @thread_array.length > 0 + # puts "Waiting for #{@thread_array.length} example threads to complete." + sleep 1 #0.1 + end + end + + # Method creates a file based mutex to prevent problems due to parallel + # access to a global / shared value + def set_lock + (@lock = File.new(@fname,"r+")).flock(File::LOCK_EX) + end + + # Method releases the file based mutex + def release_lock + @lock.flock(File::LOCK_UN) + end + end + end +end \ No newline at end of file diff --git a/lib/rspec/core/option_parser.rb b/lib/rspec/core/option_parser.rb index 03e91fefb0..fb9c3b6859 100644 --- a/lib/rspec/core/option_parser.rb +++ b/lib/rspec/core/option_parser.rb @@ -137,6 +137,20 @@ def parser(options) $VERBOSE = true end + parser.on('--parallel-test NUMBER', 'Run the tests with the specified number of parallel threads') do |n| + options[:parallel_test] = if !n.nil? + begin + Integer(n) + rescue ArgumentError + RSpec.warning "Non integer specified as number of parallel threads, seperate " + + "your path from options with a space e.g. " + + "`rspec --parallel-test #{n}`", + :call_site => nil + 1 + end + end + end + parser.separator <<-FILTERING **** Filtering/tags **** diff --git a/lib/rspec/core/runner.rb b/lib/rspec/core/runner.rb index 5c452fd57d..212602a14d 100644 --- a/lib/rspec/core/runner.rb +++ b/lib/rspec/core/runner.rb @@ -57,17 +57,21 @@ def self.invoke def self.run(args, err=$stderr, out=$stdout) trap_interrupt options = ConfigurationOptions.new(args) + run_local = !options.options[:drb] - if options.options[:drb] + if !run_local require 'rspec/core/drb' begin DRbRunner.new(options).run(err, out) rescue DRb::DRbConnError err.puts "No DRb server is running. Running in local process instead ..." - new(options).run(err, out) + run_local = true end - else - new(options).run(err, out) + end + + if run_local + num_threads = options.options[:parallel_test] || 1 + new(options).run(err, out, num_threads) end end @@ -81,9 +85,10 @@ def initialize(options, configuration=RSpec.configuration, world=RSpec.world) # # @param err [IO] error stream # @param out [IO] output stream - def run(err, out) + # @param num_threads [int] maximum number of parallel threads to use + def run(err, out, num_threads=1) setup(err, out) - run_specs(@world.ordered_example_groups) + run_specs(@world.ordered_example_groups, num_threads) end # Wires together the various configuration objects and state holders. @@ -104,12 +109,28 @@ def setup(err, out) # @return [Fixnum] exit status code. 0 if all specs passed, # or the configured failure exit code (1 by default) if specs # failed. - def run_specs(example_groups) + def run_specs(example_groups, num_threads=1) @configuration.reporter.report(@world.example_count(example_groups)) do |reporter| begin hook_context = SuiteHookContext.new @configuration.hooks.run(:before, :suite, hook_context) - example_groups.map { |g| g.run(reporter) }.all? ? 0 : @configuration.failure_exit_code + # example_groups.map { |g| g.run(reporter) }.all? ? 0 : @configuration.failure_exit_code + group_threads = RSpec::Core::ExampleGroupThreadRunner.new + example_groups.map {|g| + group_threads.run(g, reporter, num_threads) + } + + # wait for example_groups to complete + group_threads.wait_for_completion + + # get results of testing now that we're done + example_groups.map { |g| + reporter.example_group_started(g) + result_for_this_group = g.succeeded? + results_for_descendants = g.ordering_strategy.order(g.children).map {|child| child.succeeded? }.all? + reporter.example_group_finished(g) + result_for_this_group && results_for_descendants + }.all? ? 0 : @configuration.failure_exit_code ensure @configuration.hooks.run(:after, :suite, hook_context) end From c2ee7ec1c7b183467ebc0e601105b39614f93e15 Mon Sep 17 00:00:00 2001 From: bicarbon8 Date: Wed, 14 May 2014 09:57:52 +0100 Subject: [PATCH 02/13] Changes based on Pull Request #1527 feedback Removing file-based mutex and replacing with Mutex class Fixing multiline blocks to use 'do' instead of '{' Removing extra comments Switched environment to POSIX git commits Fixed spacing on default method values '=' became ' = ' Removed changes to Example class in favour of using metadata for example success Switched to .all? {...} from .map {...}.all? Corrected method parameter reference from [int] to [Integer] --- lib/rspec/core/example.rb | 11 ------ lib/rspec/core/example_group.rb | 12 +++---- lib/rspec/core/example_group_thread_runner.rb | 16 +++++---- lib/rspec/core/example_thread_runner.rb | 35 ++----------------- lib/rspec/core/runner.rb | 24 +++++-------- 5 files changed, 26 insertions(+), 72 deletions(-) diff --git a/lib/rspec/core/example.rb b/lib/rspec/core/example.rb index 0a16c6fa16..97725de8c8 100644 --- a/lib/rspec/core/example.rb +++ b/lib/rspec/core/example.rb @@ -79,16 +79,6 @@ def description RSpec.configuration.format_docstrings_block.call(description) end - # @attr_accessor - # - # Holds the completion status of the example (nil if not completed) - attr_accessor :succeeded - - # Convenience method for getting success status of example - def succeeded? - @succeeded - end - # @attr_reader # # Returns the first exception raised in the context of running this @@ -156,7 +146,6 @@ def run(example_group_instance, reporter) begin run_before_example @example_group_instance.instance_exec(self, &@example_block) - @succeeded = true # this will not be set to true if a failure occurs if pending? Pending.mark_fixed! self diff --git a/lib/rspec/core/example_group.rb b/lib/rspec/core/example_group.rb index d08ab0c881..17310cd13e 100644 --- a/lib/rspec/core/example_group.rb +++ b/lib/rspec/core/example_group.rb @@ -445,7 +445,7 @@ def self.run_after_context_hooks(example_group_instance) end # Runs all the examples in this group - def self.run(reporter, num_threads=1) + def self.run(reporter, num_threads = 1) if RSpec.world.wants_to_quit RSpec.world.clear_remaining_example_groups if top_level? return @@ -456,15 +456,15 @@ def self.run(reporter, num_threads=1) run_before_context_hooks(new) example_threads = RSpec::Core::ExampleThreadRunner.new(num_threads) run_examples(reporter, example_threads) - ordering_strategy.order(children).map {|child| child.run(reporter, num_threads)} + ordering_strategy.order(children).map { |child| child.run(reporter, num_threads) } # ensure all examples in the group are done before running 'after :all' # this does NOT prevent utilization of other available threads example_threads.wait_for_completion rescue Pending::SkipDeclaredInExample => ex - for_filtered_examples(reporter) {|example| example.skip_with_exception(reporter, ex) } + for_filtered_examples(reporter) { |example| example.skip_with_exception(reporter, ex) } rescue Exception => ex RSpec.world.wants_to_quit = true if fail_fast? - for_filtered_examples(reporter) {|example| example.fail_with_exception(reporter, ex) } + for_filtered_examples(reporter) { |example| example.fail_with_exception(reporter, ex) } ensure run_after_context_hooks(new) before_context_ivars.clear @@ -475,9 +475,7 @@ def self.run(reporter, num_threads=1) # Method will report the success / failure status of this ExampleGroup # based on the success / failure status of its contained Examples def self.succeeded? - filtered_examples.map do |example| - example.succeeded? - end.all? + filtered_examples.all? { |example| example.metadata[:execution_result] == :passed } end # @private diff --git a/lib/rspec/core/example_group_thread_runner.rb b/lib/rspec/core/example_group_thread_runner.rb index 8c91632863..17b778ea82 100644 --- a/lib/rspec/core/example_group_thread_runner.rb +++ b/lib/rspec/core/example_group_thread_runner.rb @@ -5,13 +5,17 @@ class ExampleGroupThreadRunner def initialize @thread_array = [] + $mutex = $mutex || Mutex.new end - def run(examplegroup, reporter, num_threads=1) + # Method will run an [ExampleGroup] inside a [Thread] to prevent blocking + # execution. The new [Thread] is added to an array for tracking and + # will automatically remove itself when done + def run(examplegroup, reporter, num_threads = 1) @thread_array.push Thread.start { - # puts "Starting examplegroup '#{examplegroup.description}'..." - examplegroup.run(reporter, num_threads) - # puts "Examplegroup '#{examplegroup.description}' completed." + $mutex.synchronize { + examplegroup.run(reporter, num_threads) + } @thread_array.delete Thread.current # remove from local scope } end @@ -20,10 +24,8 @@ def run(examplegroup, reporter, num_threads=1) # remove themselves from the @thread_array so an empty array means they # completed def wait_for_completion - # wait for threads to complete while @thread_array.length > 0 - # puts "Waiting for #{@thread_array.length} group threads to complete." - sleep 1 #0.1 + sleep 1 end end end diff --git a/lib/rspec/core/example_thread_runner.rb b/lib/rspec/core/example_thread_runner.rb index 1384bce0f6..989b07f3f3 100644 --- a/lib/rspec/core/example_thread_runner.rb +++ b/lib/rspec/core/example_thread_runner.rb @@ -1,72 +1,43 @@ module RSpec module Core class ExampleThreadRunner - attr_accessor :num_threads, :thread_array, :fname, :lock + attr_accessor :num_threads, :thread_array def initialize(num_threads) @num_threads = num_threads # used to track the local usage of threads - # puts "Creating ExampleThreadRunner object with #{@num_threads} threads." @thread_array = [] $used_threads = $used_threads || 0 # used to track the global usage of threads - @fname = '.examplelock' - if !File.exists? @fname - File.new(@fname, "a+") { |f| f.write "lock" } - end end # Method will check global utilization of threads and if that number is # at or over the allocated maximum it will wait until a thread is available def wait_for_available_thread - # puts "Global threads in use = #{$used_threads}." - # puts "Local threads in use = #{@thread_array.length}." # wait for available thread if we've reached our global limit while $used_threads.to_i >= @num_threads.to_i - # puts "Waiting for available thread. Running = #{$used_threads}; Max = #{@num_threads}" - sleep 1 #0.1 + sleep 1 end end # Method will run the specified example within an available thread or # will wait for a thread to become available if none currently are def run(example, instance, reporter) - # puts "Setting lock for example '#{example.description}'..." - set_lock wait_for_available_thread @thread_array.push Thread.start { - # puts "Starting example '#{example.description}'..." example.run(instance, reporter) - # puts "Example '#{example.description}' completed." @thread_array.delete Thread.current # remove from local scope $used_threads -= 1 # remove from global scope } $used_threads += 1 # add to global scope - ensure - # puts "Releasing lock for example '#{example.description}'..." - release_lock - # puts "Lock for '#{example.description}' released." end # Method will wait for all threads to complete. On completion threads # remove themselves from the @thread_array so an empty array means they # completed def wait_for_completion - # wait for threads to complete while @thread_array.length > 0 - # puts "Waiting for #{@thread_array.length} example threads to complete." - sleep 1 #0.1 + sleep 1 end end - - # Method creates a file based mutex to prevent problems due to parallel - # access to a global / shared value - def set_lock - (@lock = File.new(@fname,"r+")).flock(File::LOCK_EX) - end - - # Method releases the file based mutex - def release_lock - @lock.flock(File::LOCK_UN) - end end end end \ No newline at end of file diff --git a/lib/rspec/core/runner.rb b/lib/rspec/core/runner.rb index 212602a14d..a171acdabd 100644 --- a/lib/rspec/core/runner.rb +++ b/lib/rspec/core/runner.rb @@ -59,7 +59,7 @@ def self.run(args, err=$stderr, out=$stdout) options = ConfigurationOptions.new(args) run_local = !options.options[:drb] - if !run_local + unless run_local require 'rspec/core/drb' begin DRbRunner.new(options).run(err, out) @@ -85,8 +85,8 @@ def initialize(options, configuration=RSpec.configuration, world=RSpec.world) # # @param err [IO] error stream # @param out [IO] output stream - # @param num_threads [int] maximum number of parallel threads to use - def run(err, out, num_threads=1) + # @param num_threads [Integer] maximum number of parallel threads to use + def run(err, out, num_threads = 1) setup(err, out) run_specs(@world.ordered_example_groups, num_threads) end @@ -109,28 +109,22 @@ def setup(err, out) # @return [Fixnum] exit status code. 0 if all specs passed, # or the configured failure exit code (1 by default) if specs # failed. - def run_specs(example_groups, num_threads=1) + def run_specs(example_groups, num_threads = 1) @configuration.reporter.report(@world.example_count(example_groups)) do |reporter| begin hook_context = SuiteHookContext.new @configuration.hooks.run(:before, :suite, hook_context) - # example_groups.map { |g| g.run(reporter) }.all? ? 0 : @configuration.failure_exit_code + group_threads = RSpec::Core::ExampleGroupThreadRunner.new - example_groups.map {|g| - group_threads.run(g, reporter, num_threads) - } - - # wait for example_groups to complete + example_groups.each { |g| group_threads.run(g, reporter, num_threads) } group_threads.wait_for_completion # get results of testing now that we're done - example_groups.map { |g| - reporter.example_group_started(g) + example_groups.all? do |g| result_for_this_group = g.succeeded? - results_for_descendants = g.ordering_strategy.order(g.children).map {|child| child.succeeded? }.all? - reporter.example_group_finished(g) + results_for_descendants = g.children.all? { |child| child.succeeded? } result_for_this_group && results_for_descendants - }.all? ? 0 : @configuration.failure_exit_code + end ? 0 : @configuration.failure_exit_code ensure @configuration.hooks.run(:after, :suite, hook_context) end From 65d4d6225be83bfb31c04159e69f9020be901249 Mon Sep 17 00:00:00 2001 From: bicarbon8 Date: Wed, 14 May 2014 13:12:07 +0100 Subject: [PATCH 03/13] Change to how maximum number of threads is passed to execution based on Pull Request #1527 feedback Removes need to pass 'num_threads' directly from options into execution methods and then uses config values to specify the thread max limit --- lib/rspec/core/configuration.rb | 3 ++- lib/rspec/core/configuration_options.rb | 4 ++-- lib/rspec/core/option_parser.rb | 2 +- lib/rspec/core/runner.rb | 12 +++++------- 4 files changed, 10 insertions(+), 11 deletions(-) diff --git a/lib/rspec/core/configuration.rb b/lib/rspec/core/configuration.rb index 69d82fe97f..3057cf0493 100644 --- a/lib/rspec/core/configuration.rb +++ b/lib/rspec/core/configuration.rb @@ -267,7 +267,7 @@ def treat_symbols_as_metadata_keys_with_true_values=(value) # @private add_setting :expecting_with_rspec # @private - attr_accessor :filter_manager + attr_accessor :filter_manager, :thread_maximum # @private attr_reader :backtrace_formatter, :ordering_manager @@ -302,6 +302,7 @@ def initialize @requires = [] @libs = [] @derived_metadata_blocks = [] + @thread_maximum = 1 end # @private diff --git a/lib/rspec/core/configuration_options.rb b/lib/rspec/core/configuration_options.rb index d0b114fa3c..503a4b6932 100644 --- a/lib/rspec/core/configuration_options.rb +++ b/lib/rspec/core/configuration_options.rb @@ -53,10 +53,10 @@ def organize_options UNFORCED_OPTIONS = [ :requires, :profile, :drb, :libs, :files_or_directories_to_run, - :full_description, :full_backtrace, :tty, :parallel_test + :full_description, :full_backtrace, :tty, :thread_maximum ].to_set - UNPROCESSABLE_OPTIONS = [:formatters, :parallel_test].to_set + UNPROCESSABLE_OPTIONS = [:formatters].to_set def force?(key) !UNFORCED_OPTIONS.include?(key) diff --git a/lib/rspec/core/option_parser.rb b/lib/rspec/core/option_parser.rb index fb9c3b6859..aa8c867be5 100644 --- a/lib/rspec/core/option_parser.rb +++ b/lib/rspec/core/option_parser.rb @@ -138,7 +138,7 @@ def parser(options) end parser.on('--parallel-test NUMBER', 'Run the tests with the specified number of parallel threads') do |n| - options[:parallel_test] = if !n.nil? + options[:thread_maximum] = if !n.nil? begin Integer(n) rescue ArgumentError diff --git a/lib/rspec/core/runner.rb b/lib/rspec/core/runner.rb index a171acdabd..c89b720f58 100644 --- a/lib/rspec/core/runner.rb +++ b/lib/rspec/core/runner.rb @@ -70,8 +70,7 @@ def self.run(args, err=$stderr, out=$stdout) end if run_local - num_threads = options.options[:parallel_test] || 1 - new(options).run(err, out, num_threads) + new(options).run(err, out) end end @@ -85,10 +84,9 @@ def initialize(options, configuration=RSpec.configuration, world=RSpec.world) # # @param err [IO] error stream # @param out [IO] output stream - # @param num_threads [Integer] maximum number of parallel threads to use - def run(err, out, num_threads = 1) + def run(err, out) setup(err, out) - run_specs(@world.ordered_example_groups, num_threads) + run_specs(@world.ordered_example_groups) end # Wires together the various configuration objects and state holders. @@ -109,14 +107,14 @@ def setup(err, out) # @return [Fixnum] exit status code. 0 if all specs passed, # or the configured failure exit code (1 by default) if specs # failed. - def run_specs(example_groups, num_threads = 1) + def run_specs(example_groups) @configuration.reporter.report(@world.example_count(example_groups)) do |reporter| begin hook_context = SuiteHookContext.new @configuration.hooks.run(:before, :suite, hook_context) group_threads = RSpec::Core::ExampleGroupThreadRunner.new - example_groups.each { |g| group_threads.run(g, reporter, num_threads) } + example_groups.each { |g| group_threads.run(g, reporter, @configuration.thread_maximum) } group_threads.wait_for_completion # get results of testing now that we're done From 148e08161efc1c75c0c37aa9c7d1136d95038dd7 Mon Sep 17 00:00:00 2001 From: bicarbon8 Date: Wed, 14 May 2014 13:39:58 +0100 Subject: [PATCH 04/13] Moving mutex down to smallest possible scope. Prior placement was too high up the thread chain resulting in less optimal thread utilization --- lib/rspec/core/example_group.rb | 4 +++- lib/rspec/core/example_group_thread_runner.rb | 5 +---- lib/rspec/core/runner.rb | 1 + 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/lib/rspec/core/example_group.rb b/lib/rspec/core/example_group.rb index 17310cd13e..c52531a09b 100644 --- a/lib/rspec/core/example_group.rb +++ b/lib/rspec/core/example_group.rb @@ -500,7 +500,9 @@ def self.run_examples(reporter, threads) next if RSpec.world.wants_to_quit instance = new set_ivars(instance, before_context_ivars) - threads.run(example, instance, reporter) + $mutex.synchronize { + threads.run(example, instance, reporter) + } end end diff --git a/lib/rspec/core/example_group_thread_runner.rb b/lib/rspec/core/example_group_thread_runner.rb index 17b778ea82..e846c0d5ce 100644 --- a/lib/rspec/core/example_group_thread_runner.rb +++ b/lib/rspec/core/example_group_thread_runner.rb @@ -5,7 +5,6 @@ class ExampleGroupThreadRunner def initialize @thread_array = [] - $mutex = $mutex || Mutex.new end # Method will run an [ExampleGroup] inside a [Thread] to prevent blocking @@ -13,9 +12,7 @@ def initialize # will automatically remove itself when done def run(examplegroup, reporter, num_threads = 1) @thread_array.push Thread.start { - $mutex.synchronize { - examplegroup.run(reporter, num_threads) - } + examplegroup.run(reporter, num_threads) @thread_array.delete Thread.current # remove from local scope } end diff --git a/lib/rspec/core/runner.rb b/lib/rspec/core/runner.rb index c89b720f58..1eacb60b27 100644 --- a/lib/rspec/core/runner.rb +++ b/lib/rspec/core/runner.rb @@ -108,6 +108,7 @@ def setup(err, out) # or the configured failure exit code (1 by default) if specs # failed. def run_specs(example_groups) + $mutex = Mutex.new @configuration.reporter.report(@world.example_count(example_groups)) do |reporter| begin hook_context = SuiteHookContext.new From c898ab00b693d383d95c5c7637f83be933a7baa4 Mon Sep 17 00:00:00 2001 From: bicarbon8 Date: Wed, 14 May 2014 13:47:45 +0100 Subject: [PATCH 05/13] Changing how execution occurs depending on inclusion of the --parallel-test parameter based on feedback in Pull Request #1527 This reverts back to the standard execution methods for non-parallel execution and then uses custom methods when the --parallel-test argument is used to allow for late binding of the require 'thread' call --- features/command_line/parallel_test.feature | 40 ++++++++++++++++ lib/rspec/core.rb | 2 - lib/rspec/core/example_group.rb | 46 +++++++++++++++++-- lib/rspec/core/example_group_thread_runner.rb | 2 +- lib/rspec/core/option_parser.rb | 2 +- lib/rspec/core/runner.rb | 25 +++++++++- 6 files changed, 108 insertions(+), 9 deletions(-) create mode 100644 features/command_line/parallel_test.feature diff --git a/features/command_line/parallel_test.feature b/features/command_line/parallel_test.feature new file mode 100644 index 0000000000..2968bd36fc --- /dev/null +++ b/features/command_line/parallel_test.feature @@ -0,0 +1,40 @@ +Feature: `--parallel-test` option + + Use the `--parallel-test` option to have RSpec run using multiple threads in parallel + + Scenario: Using `--parallel-test 3` + Given a file named "spec/parallel_test_spec.rb" with: + """ruby + RSpec.configure do |c| + c.before(:suite) { puts "before suite" } + c.after(:suite) { puts "after suite" } + end + + RSpec.describe "parallel run" do + before(:context) { puts "before context" } + before(:example) { puts "before example" } + + it "thread 0 example" do + fail + end + + it "thread 1 example" do + pass + end + + it "thread 2 example" do + pass + end + + after(:example) { puts "after example" } + after(:context) { puts "after context" } + end + """ + When I run `rspec --parallel-test 3` + Then the output should contain "3 examples, 1 failure" + And the output should contain "before suite" + And the output should contain "after suite" + And the output should contain "before context" + And the output should contain "after context" + And the output should contain "before example" + And the output should contain "after example" diff --git a/lib/rspec/core.rb b/lib/rspec/core.rb index c45aabca84..67117dfa2c 100644 --- a/lib/rspec/core.rb +++ b/lib/rspec/core.rb @@ -1,6 +1,4 @@ $_rspec_core_load_started_at = Time.now -require 'thread' - require 'rbconfig' require "rspec/support" diff --git a/lib/rspec/core/example_group.rb b/lib/rspec/core/example_group.rb index c52531a09b..d5d9fab0be 100644 --- a/lib/rspec/core/example_group.rb +++ b/lib/rspec/core/example_group.rb @@ -445,7 +445,32 @@ def self.run_after_context_hooks(example_group_instance) end # Runs all the examples in this group - def self.run(reporter, num_threads = 1) + def self.run(reporter) + if RSpec.world.wants_to_quit + RSpec.world.clear_remaining_example_groups if top_level? + return + end + reporter.example_group_started(self) + + begin + run_before_context_hooks(new) + result_for_this_group = run_examples(reporter) + results_for_descendants = ordering_strategy.order(children).map { |child| child.run(reporter) }.all? + result_for_this_group && results_for_descendants + rescue Pending::SkipDeclaredInExample => ex + for_filtered_examples(reporter) {|example| example.skip_with_exception(reporter, ex) } + rescue Exception => ex + RSpec.world.wants_to_quit = true if fail_fast? + for_filtered_examples(reporter) {|example| example.fail_with_exception(reporter, ex) } + ensure + run_after_context_hooks(new) + before_context_ivars.clear + reporter.example_group_finished(self) + end + end + + # Runs all the examples in this group + def self.run_parallel(reporter, num_threads = 1) if RSpec.world.wants_to_quit RSpec.world.clear_remaining_example_groups if top_level? return @@ -455,8 +480,8 @@ def self.run(reporter, num_threads = 1) begin run_before_context_hooks(new) example_threads = RSpec::Core::ExampleThreadRunner.new(num_threads) - run_examples(reporter, example_threads) - ordering_strategy.order(children).map { |child| child.run(reporter, num_threads) } + run_examples_parallel(reporter, example_threads) + ordering_strategy.order(children).map { |child| child.run_parallel(reporter, num_threads) } # ensure all examples in the group are done before running 'after :all' # this does NOT prevent utilization of other available threads example_threads.wait_for_completion @@ -495,7 +520,20 @@ def self.ordering_strategy end # @private - def self.run_examples(reporter, threads) + def self.run_examples(reporter) + + ordering_strategy.order(filtered_examples).map do |example| + next if RSpec.world.wants_to_quit + instance = new + set_ivars(instance, before_context_ivars) + succeeded = example.run(instance, reporter) + RSpec.world.wants_to_quit = true if fail_fast? && !succeeded + succeeded + end.all? + end + + # @private + def self.run_examples_parallel(reporter, threads) ordering_strategy.order(filtered_examples).map do |example| next if RSpec.world.wants_to_quit instance = new diff --git a/lib/rspec/core/example_group_thread_runner.rb b/lib/rspec/core/example_group_thread_runner.rb index e846c0d5ce..4245145483 100644 --- a/lib/rspec/core/example_group_thread_runner.rb +++ b/lib/rspec/core/example_group_thread_runner.rb @@ -12,7 +12,7 @@ def initialize # will automatically remove itself when done def run(examplegroup, reporter, num_threads = 1) @thread_array.push Thread.start { - examplegroup.run(reporter, num_threads) + examplegroup.run_parallel(reporter, num_threads) @thread_array.delete Thread.current # remove from local scope } end diff --git a/lib/rspec/core/option_parser.rb b/lib/rspec/core/option_parser.rb index aa8c867be5..c8da924bdc 100644 --- a/lib/rspec/core/option_parser.rb +++ b/lib/rspec/core/option_parser.rb @@ -137,7 +137,7 @@ def parser(options) $VERBOSE = true end - parser.on('--parallel-test NUMBER', 'Run the tests with the specified number of parallel threads') do |n| + parser.on('--parallel-test NUMBER', 'Run the tests with the specified number of parallel threads (default: 1).') do |n| options[:thread_maximum] = if !n.nil? begin Integer(n) diff --git a/lib/rspec/core/runner.rb b/lib/rspec/core/runner.rb index 1eacb60b27..02409c3258 100644 --- a/lib/rspec/core/runner.rb +++ b/lib/rspec/core/runner.rb @@ -86,7 +86,12 @@ def initialize(options, configuration=RSpec.configuration, world=RSpec.world) # @param out [IO] output stream def run(err, out) setup(err, out) - run_specs(@world.ordered_example_groups) + if @options.options[:thread_maximum].nil? + run_specs(@world.ordered_example_groups) + else + require 'thread' + run_specs_parallel(@world.ordered_example_groups) + end end # Wires together the various configuration objects and state holders. @@ -108,6 +113,24 @@ def setup(err, out) # or the configured failure exit code (1 by default) if specs # failed. def run_specs(example_groups) + @configuration.reporter.report(@world.example_count(example_groups)) do |reporter| + begin + hook_context = SuiteHookContext.new + @configuration.hooks.run(:before, :suite, hook_context) + example_groups.map { |g| g.run(reporter) }.all? ? 0 : @configuration.failure_exit_code + ensure + @configuration.hooks.run(:after, :suite, hook_context) + end + end + end + + # Runs the provided example groups in parallel. + # + # @param example_groups [Array] groups to run + # @return [Fixnum] exit status code. 0 if all specs passed, + # or the configured failure exit code (1 by default) if specs + # failed. + def run_specs_parallel(example_groups) $mutex = Mutex.new @configuration.reporter.report(@world.example_count(example_groups)) do |reporter| begin From 708e869b3d019901be8cc0fbcb9b680b60add27c Mon Sep 17 00:00:00 2001 From: bicarbon8 Date: Thu, 15 May 2014 10:42:07 +0100 Subject: [PATCH 06/13] Removing global mutex and thread max values and instead passing them through the calls fixing line endings to use POSIX style (hopefully) Removing inline comments from code Reverted runner.rb self.run method back to original state Modified how the result of testing is collected in runner.rb run_specs_parallel so now it will only return an error code if there is an exception in executing the example or its before or after blocks --- lib/rspec/core/example_group.rb | 19 ++++++------------- lib/rspec/core/example_group_thread_runner.rb | 14 ++++++++------ lib/rspec/core/example_thread_runner.rb | 11 +++++------ lib/rspec/core/runner.rb | 19 +++++++------------ 4 files changed, 26 insertions(+), 37 deletions(-) diff --git a/lib/rspec/core/example_group.rb b/lib/rspec/core/example_group.rb index d5d9fab0be..cbbe94b67e 100644 --- a/lib/rspec/core/example_group.rb +++ b/lib/rspec/core/example_group.rb @@ -470,7 +470,7 @@ def self.run(reporter) end # Runs all the examples in this group - def self.run_parallel(reporter, num_threads = 1) + def self.run_parallel(reporter, num_threads = 1, mutex) if RSpec.world.wants_to_quit RSpec.world.clear_remaining_example_groups if top_level? return @@ -480,8 +480,8 @@ def self.run_parallel(reporter, num_threads = 1) begin run_before_context_hooks(new) example_threads = RSpec::Core::ExampleThreadRunner.new(num_threads) - run_examples_parallel(reporter, example_threads) - ordering_strategy.order(children).map { |child| child.run_parallel(reporter, num_threads) } + run_examples_parallel(reporter, example_threads, mutex) + ordering_strategy.order(children).map { |child| child.run_parallel(reporter, num_threads, mutex) } # ensure all examples in the group are done before running 'after :all' # this does NOT prevent utilization of other available threads example_threads.wait_for_completion @@ -497,12 +497,6 @@ def self.run_parallel(reporter, num_threads = 1) end end - # Method will report the success / failure status of this ExampleGroup - # based on the success / failure status of its contained Examples - def self.succeeded? - filtered_examples.all? { |example| example.metadata[:execution_result] == :passed } - end - # @private def self.ordering_strategy order = metadata.fetch(:order, :global) @@ -521,7 +515,6 @@ def self.ordering_strategy # @private def self.run_examples(reporter) - ordering_strategy.order(filtered_examples).map do |example| next if RSpec.world.wants_to_quit instance = new @@ -533,14 +526,14 @@ def self.run_examples(reporter) end # @private - def self.run_examples_parallel(reporter, threads) + def self.run_examples_parallel(reporter, threads, mutex) ordering_strategy.order(filtered_examples).map do |example| next if RSpec.world.wants_to_quit instance = new set_ivars(instance, before_context_ivars) - $mutex.synchronize { + mutex.synchronize do threads.run(example, instance, reporter) - } + end end end diff --git a/lib/rspec/core/example_group_thread_runner.rb b/lib/rspec/core/example_group_thread_runner.rb index 4245145483..45784058ac 100644 --- a/lib/rspec/core/example_group_thread_runner.rb +++ b/lib/rspec/core/example_group_thread_runner.rb @@ -1,19 +1,21 @@ module RSpec module Core class ExampleGroupThreadRunner - attr_accessor :thread_array + attr_accessor :thread_array, :max_threads, :mutex - def initialize + def initialize(max_threads = 1, mutex = Mutex.new) + @max_threads = max_threads + @mutex = mutex @thread_array = [] end # Method will run an [ExampleGroup] inside a [Thread] to prevent blocking # execution. The new [Thread] is added to an array for tracking and # will automatically remove itself when done - def run(examplegroup, reporter, num_threads = 1) + def run(examplegroup, reporter) @thread_array.push Thread.start { - examplegroup.run_parallel(reporter, num_threads) - @thread_array.delete Thread.current # remove from local scope + examplegroup.run_parallel(reporter, @max_threads, @mutex) + @thread_array.delete Thread.current } end @@ -27,4 +29,4 @@ def wait_for_completion end end end -end \ No newline at end of file +end diff --git a/lib/rspec/core/example_thread_runner.rb b/lib/rspec/core/example_thread_runner.rb index 989b07f3f3..76b4c5c256 100644 --- a/lib/rspec/core/example_thread_runner.rb +++ b/lib/rspec/core/example_thread_runner.rb @@ -4,15 +4,14 @@ class ExampleThreadRunner attr_accessor :num_threads, :thread_array def initialize(num_threads) - @num_threads = num_threads # used to track the local usage of threads + @num_threads = num_threads @thread_array = [] - $used_threads = $used_threads || 0 # used to track the global usage of threads + $used_threads = $used_threads || 0 end # Method will check global utilization of threads and if that number is # at or over the allocated maximum it will wait until a thread is available def wait_for_available_thread - # wait for available thread if we've reached our global limit while $used_threads.to_i >= @num_threads.to_i sleep 1 end @@ -25,9 +24,9 @@ def run(example, instance, reporter) @thread_array.push Thread.start { example.run(instance, reporter) @thread_array.delete Thread.current # remove from local scope - $used_threads -= 1 # remove from global scope + $used_threads -= 1 } - $used_threads += 1 # add to global scope + $used_threads += 1 end # Method will wait for all threads to complete. On completion threads @@ -40,4 +39,4 @@ def wait_for_completion end end end -end \ No newline at end of file +end diff --git a/lib/rspec/core/runner.rb b/lib/rspec/core/runner.rb index 02409c3258..0aef9b49ca 100644 --- a/lib/rspec/core/runner.rb +++ b/lib/rspec/core/runner.rb @@ -57,19 +57,16 @@ def self.invoke def self.run(args, err=$stderr, out=$stdout) trap_interrupt options = ConfigurationOptions.new(args) - run_local = !options.options[:drb] - unless run_local + if options.options[:drb] require 'rspec/core/drb' begin DRbRunner.new(options).run(err, out) rescue DRb::DRbConnError err.puts "No DRb server is running. Running in local process instead ..." - run_local = true + new(options).run(err, out) end - end - - if run_local + else new(options).run(err, out) end end @@ -131,20 +128,18 @@ def run_specs(example_groups) # or the configured failure exit code (1 by default) if specs # failed. def run_specs_parallel(example_groups) - $mutex = Mutex.new @configuration.reporter.report(@world.example_count(example_groups)) do |reporter| begin hook_context = SuiteHookContext.new @configuration.hooks.run(:before, :suite, hook_context) - group_threads = RSpec::Core::ExampleGroupThreadRunner.new - example_groups.each { |g| group_threads.run(g, reporter, @configuration.thread_maximum) } + group_threads = RSpec::Core::ExampleGroupThreadRunner.new(@configuration.thread_maximum) + example_groups.each { |g| group_threads.run(g, reporter) } group_threads.wait_for_completion - # get results of testing now that we're done example_groups.all? do |g| - result_for_this_group = g.succeeded? - results_for_descendants = g.children.all? { |child| child.succeeded? } + result_for_this_group = g.filtered_examples.all? { |example| example.metadata[:execution_result].exception.nil? } + results_for_descendants = g.children.all? { |child| child.filtered_examples.all? { |example| example.metadata[:execution_result].exception.nil? } } result_for_this_group && results_for_descendants end ? 0 : @configuration.failure_exit_code ensure From 28a18e5176f733b5f3038ba6c09ad3a918c4c482 Mon Sep 17 00:00:00 2001 From: bicarbon8 Date: Thu, 15 May 2014 11:21:42 +0100 Subject: [PATCH 07/13] removes additional global missed in last commit ($used_threads) and now passes the variable by reference to the areas where needed. --- lib/rspec/core/example_group.rb | 6 +++--- lib/rspec/core/example_group_thread_runner.rb | 7 ++++--- lib/rspec/core/example_thread_runner.rb | 14 +++++++------- 3 files changed, 14 insertions(+), 13 deletions(-) diff --git a/lib/rspec/core/example_group.rb b/lib/rspec/core/example_group.rb index cbbe94b67e..d1ebd7d40d 100644 --- a/lib/rspec/core/example_group.rb +++ b/lib/rspec/core/example_group.rb @@ -470,7 +470,7 @@ def self.run(reporter) end # Runs all the examples in this group - def self.run_parallel(reporter, num_threads = 1, mutex) + def self.run_parallel(reporter, num_threads = 1, mutex, used_threads) if RSpec.world.wants_to_quit RSpec.world.clear_remaining_example_groups if top_level? return @@ -479,9 +479,9 @@ def self.run_parallel(reporter, num_threads = 1, mutex) begin run_before_context_hooks(new) - example_threads = RSpec::Core::ExampleThreadRunner.new(num_threads) + example_threads = RSpec::Core::ExampleThreadRunner.new(num_threads, used_threads) run_examples_parallel(reporter, example_threads, mutex) - ordering_strategy.order(children).map { |child| child.run_parallel(reporter, num_threads, mutex) } + ordering_strategy.order(children).map { |child| child.run_parallel(reporter, num_threads, mutex, used_threads) } # ensure all examples in the group are done before running 'after :all' # this does NOT prevent utilization of other available threads example_threads.wait_for_completion diff --git a/lib/rspec/core/example_group_thread_runner.rb b/lib/rspec/core/example_group_thread_runner.rb index 45784058ac..ed573c1d80 100644 --- a/lib/rspec/core/example_group_thread_runner.rb +++ b/lib/rspec/core/example_group_thread_runner.rb @@ -1,11 +1,12 @@ module RSpec module Core class ExampleGroupThreadRunner - attr_accessor :thread_array, :max_threads, :mutex + attr_accessor :thread_array, :max_threads, :mutex, :used_threads - def initialize(max_threads = 1, mutex = Mutex.new) + def initialize(max_threads = 1, mutex = Mutex.new, used_threads = 0) @max_threads = max_threads @mutex = mutex + @used_threads = used_threads @thread_array = [] end @@ -14,7 +15,7 @@ def initialize(max_threads = 1, mutex = Mutex.new) # will automatically remove itself when done def run(examplegroup, reporter) @thread_array.push Thread.start { - examplegroup.run_parallel(reporter, @max_threads, @mutex) + examplegroup.run_parallel(reporter, @max_threads, @mutex, @used_threads) @thread_array.delete Thread.current } end diff --git a/lib/rspec/core/example_thread_runner.rb b/lib/rspec/core/example_thread_runner.rb index 76b4c5c256..9b7f0b1773 100644 --- a/lib/rspec/core/example_thread_runner.rb +++ b/lib/rspec/core/example_thread_runner.rb @@ -1,18 +1,18 @@ module RSpec module Core class ExampleThreadRunner - attr_accessor :num_threads, :thread_array + attr_accessor :num_threads, :thread_array, :used_threads - def initialize(num_threads) + def initialize(num_threads, used_threads) @num_threads = num_threads @thread_array = [] - $used_threads = $used_threads || 0 + @used_threads = used_threads end # Method will check global utilization of threads and if that number is # at or over the allocated maximum it will wait until a thread is available def wait_for_available_thread - while $used_threads.to_i >= @num_threads.to_i + while @used_threads.to_i >= @num_threads.to_i sleep 1 end end @@ -21,12 +21,12 @@ def wait_for_available_thread # will wait for a thread to become available if none currently are def run(example, instance, reporter) wait_for_available_thread - @thread_array.push Thread.start { + @thread_array.push Thread.start { example.run(instance, reporter) @thread_array.delete Thread.current # remove from local scope - $used_threads -= 1 + @used_threads -= 1 } - $used_threads += 1 + @used_threads += 1 end # Method will wait for all threads to complete. On completion threads From 3ace928c7d786ec12f665944b39e8476c78efa5d Mon Sep 17 00:00:00 2001 From: bicarbon8 Date: Thu, 15 May 2014 17:12:58 +0100 Subject: [PATCH 08/13] Adding rspec tests for parallel execution to help increase the code coverage metrics --- spec/command_line/parallel_spec.rb | 85 ++++++++++++++++++++++++++++++ 1 file changed, 85 insertions(+) create mode 100644 spec/command_line/parallel_spec.rb diff --git a/spec/command_line/parallel_spec.rb b/spec/command_line/parallel_spec.rb new file mode 100644 index 0000000000..77942bcf42 --- /dev/null +++ b/spec/command_line/parallel_spec.rb @@ -0,0 +1,85 @@ +require 'spec_helper' + + + +RSpec.describe 'command line', :ui, :slow do + before :each do + @out = StringIO.new + write_file 'spec/parallel_spec.rb', """ + RSpec.describe \"parallel run\" do + it \"thread 0 example\" do + sleep 1 + fail + end + + it \"thread 1 example\" do + sleep 1 + pass + end + + it \"thread 2 example\" do + sleep 1 + pass + end + + it \"thread 3 example\" do + sleep 1 + fail + end + + it \"thread 4 example\" do + sleep 1 + pass + end + + it \"thread 5 example\" do + sleep 1 + pass + end + end + """ + end + + describe '--parallel-test' do + it '1 thread' do + run_command 'spec/parallel_spec.rb --parallel-test 1' + output_str = @out.string + validate_output(output_str) + seconds = get_seconds(output_str).to_i + expect(seconds).to be >= 6 + end + + it '3 threads' do + run_command 'spec/parallel_spec.rb --parallel-test 3' + output_str = @out.string + validate_output(output_str) + seconds = get_seconds(output_str).to_i + expect(seconds).to be >= 3 + expect(seconds).to be < 6 + end + + it '6 threads' do + run_command 'spec/parallel_spec.rb --parallel-test 6' + output_str = @out.string + validate_output(output_str) + seconds = get_seconds(output_str).to_i + expect(seconds).to be >= 1 + expect(seconds).to be < 3 + end + end + + def run_command(cmd) + in_current_dir do + RSpec::Core::Runner.run(cmd.split, @out, @out) + end + end + + def validate_output(output_str) + expect(output_str).to include("6 examples, 2 failures"), "output_str: #{output_str}" + expect(output_str).to include("Finished in "), "output_str: #{output_str}" + end + + def get_seconds(output_str) + return output_str[/Finished in (?.*) second/, "match"] + end +end From 54b876e77c229eb203888f95b055e6a6b42c5b37 Mon Sep 17 00:00:00 2001 From: bicarbon8 Date: Thu, 15 May 2014 17:22:22 +0100 Subject: [PATCH 09/13] adding a 1 second buffer to the expected duration to account for speed of build machine --- spec/command_line/parallel_spec.rb | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/spec/command_line/parallel_spec.rb b/spec/command_line/parallel_spec.rb index 77942bcf42..3f5fcecac7 100644 --- a/spec/command_line/parallel_spec.rb +++ b/spec/command_line/parallel_spec.rb @@ -54,8 +54,8 @@ output_str = @out.string validate_output(output_str) seconds = get_seconds(output_str).to_i - expect(seconds).to be >= 3 - expect(seconds).to be < 6 + expect(seconds).to be >= 2 + expect(seconds).to be < 7 end it '6 threads' do @@ -63,8 +63,8 @@ output_str = @out.string validate_output(output_str) seconds = get_seconds(output_str).to_i - expect(seconds).to be >= 1 - expect(seconds).to be < 3 + expect(seconds).to be >= 0 + expect(seconds).to be < 4 end end From 060cb34acf6516da8341e8e6a7e8fa16901d9ea9 Mon Sep 17 00:00:00 2001 From: bicarbon8 Date: Thu, 15 May 2014 17:38:00 +0100 Subject: [PATCH 10/13] correcting additional test failures pertaining to default value handling in method call and undefined variable in cucumber test --- features/command_line/parallel_test.feature | 6 +++--- lib/rspec/core/example_group.rb | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/features/command_line/parallel_test.feature b/features/command_line/parallel_test.feature index 2968bd36fc..e3e620ccec 100644 --- a/features/command_line/parallel_test.feature +++ b/features/command_line/parallel_test.feature @@ -15,15 +15,15 @@ Feature: `--parallel-test` option before(:example) { puts "before example" } it "thread 0 example" do - fail + expect(true).to be(false) end it "thread 1 example" do - pass + expect(true).to be(true) end it "thread 2 example" do - pass + expect(true).to be(true) end after(:example) { puts "after example" } diff --git a/lib/rspec/core/example_group.rb b/lib/rspec/core/example_group.rb index d1ebd7d40d..135b1de95c 100644 --- a/lib/rspec/core/example_group.rb +++ b/lib/rspec/core/example_group.rb @@ -470,7 +470,7 @@ def self.run(reporter) end # Runs all the examples in this group - def self.run_parallel(reporter, num_threads = 1, mutex, used_threads) + def self.run_parallel(reporter, num_threads, mutex, used_threads) if RSpec.world.wants_to_quit RSpec.world.clear_remaining_example_groups if top_level? return From 9603ae4529a7e1e6a5b08e34e7d202e0a76a58a9 Mon Sep 17 00:00:00 2001 From: bicarbon8 Date: Thu, 15 May 2014 18:03:44 +0100 Subject: [PATCH 11/13] Adding YARD documentation to the new class files to ensure 100% documentation coverage --- lib/rspec/core/example_group_thread_runner.rb | 16 ++++++++++++++++ lib/rspec/core/example_thread_runner.rb | 13 +++++++++++++ 2 files changed, 29 insertions(+) diff --git a/lib/rspec/core/example_group_thread_runner.rb b/lib/rspec/core/example_group_thread_runner.rb index ed573c1d80..c80dada26a 100644 --- a/lib/rspec/core/example_group_thread_runner.rb +++ b/lib/rspec/core/example_group_thread_runner.rb @@ -1,8 +1,21 @@ module RSpec module Core + # ExampleGroupThreadRunner is a class used to execute [ExampleGroup] + # classes in parallel as part of rspec-core. When running in parallel + # the order of example groups will not be honoured. + # This class is used to ensure that we have a way of keeping track of + # the number of threads being created and preventing utilization of + # more than the specified number + # Additionally, this class will contain a mutex used to prevent access + # to shared variables within sub-threads class ExampleGroupThreadRunner attr_accessor :thread_array, :max_threads, :mutex, :used_threads + # Creates a new instance of ExampleGroupThreadRunner. + # @param max_threads [Integer] the maximum limit of threads that can be used + # @param mutex [Mutex] a semaphore used to prevent access to shared variables in + # sub-threads such as those used by [ExampleThreadRunner] + # @param used_threads [Integer] the current number of threads being used def initialize(max_threads = 1, mutex = Mutex.new, used_threads = 0) @max_threads = max_threads @mutex = mutex @@ -13,6 +26,9 @@ def initialize(max_threads = 1, mutex = Mutex.new, used_threads = 0) # Method will run an [ExampleGroup] inside a [Thread] to prevent blocking # execution. The new [Thread] is added to an array for tracking and # will automatically remove itself when done + # @param examplegroup [ExampleGroup] the group to be run inside a [Thread] + # @param reporter [Reporter] the passed in reporting class used for + # tracking def run(examplegroup, reporter) @thread_array.push Thread.start { examplegroup.run_parallel(reporter, @max_threads, @mutex, @used_threads) diff --git a/lib/rspec/core/example_thread_runner.rb b/lib/rspec/core/example_thread_runner.rb index 9b7f0b1773..f2296f7e22 100644 --- a/lib/rspec/core/example_thread_runner.rb +++ b/lib/rspec/core/example_thread_runner.rb @@ -1,8 +1,17 @@ module RSpec module Core + # ExampleThreadRunner is a class used to execute [Example] classes in + # parallel as part of rspec-core. When running in parallel the order + # of examples will not be honoured. + # This class is used to ensure that we have a way of keeping track of + # the number of threads being created and preventing utilization of + # more than the specified number class ExampleThreadRunner attr_accessor :num_threads, :thread_array, :used_threads + # Creates a new instance of ExampleThreadRunner. + # @param num_threads [Integer] the maximum limit of threads that can be used + # @param used_threads [Integer] the current number of threads being used def initialize(num_threads, used_threads) @num_threads = num_threads @thread_array = [] @@ -19,6 +28,10 @@ def wait_for_available_thread # Method will run the specified example within an available thread or # will wait for a thread to become available if none currently are + # @param example [Example] the example to be executed in a [Thread] + # @param instance the instance of an ExampleGroup subclass + # @param reporter [Reporter] the passed in reporting class used for + # tracking def run(example, instance, reporter) wait_for_available_thread @thread_array.push Thread.start { From 5145ed9821373cde224ea7c3422bafd16e54d2ac Mon Sep 17 00:00:00 2001 From: bicarbon8 Date: Thu, 15 May 2014 21:26:17 +0100 Subject: [PATCH 12/13] modifying regex string matcher to be compatible with older versions of ruby --- spec/command_line/parallel_spec.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spec/command_line/parallel_spec.rb b/spec/command_line/parallel_spec.rb index 3f5fcecac7..a1478f7798 100644 --- a/spec/command_line/parallel_spec.rb +++ b/spec/command_line/parallel_spec.rb @@ -80,6 +80,6 @@ def validate_output(output_str) end def get_seconds(output_str) - return output_str[/Finished in (?.*) second/, "match"] + return output_str[/Finished in (.*) second/, 1] end end From 38205dfeba555fb69cb44ebbe1dc07dffbcb85bd Mon Sep 17 00:00:00 2001 From: bicarbon8 Date: Fri, 16 May 2014 12:38:58 +0100 Subject: [PATCH 13/13] implementing changes based on feedback on PR #1527 changed "examplegroup" variable to be "example_group" modified wait_for_completion methods in example_thread_runner.rb and example_group_thread_runner.rb to join to the sub-threads instead of polling modified wait_for_available_thead method in example_thread_runner.rb to only sleep for 0.1 seconds instead of 1 second to reduce wait overhead while polling for an available thread to run within changed doublequotes to singlequotes in "spec/command_line/parallel_spec.rb" to avoid the need for escaping --- lib/rspec/core/example_group_thread_runner.rb | 10 +++++----- lib/rspec/core/example_thread_runner.rb | 6 +++--- spec/command_line/parallel_spec.rb | 16 +++++++--------- 3 files changed, 15 insertions(+), 17 deletions(-) diff --git a/lib/rspec/core/example_group_thread_runner.rb b/lib/rspec/core/example_group_thread_runner.rb index c80dada26a..2ee8240f94 100644 --- a/lib/rspec/core/example_group_thread_runner.rb +++ b/lib/rspec/core/example_group_thread_runner.rb @@ -26,12 +26,12 @@ def initialize(max_threads = 1, mutex = Mutex.new, used_threads = 0) # Method will run an [ExampleGroup] inside a [Thread] to prevent blocking # execution. The new [Thread] is added to an array for tracking and # will automatically remove itself when done - # @param examplegroup [ExampleGroup] the group to be run inside a [Thread] + # @param example_group [ExampleGroup] the group to be run inside a [Thread] # @param reporter [Reporter] the passed in reporting class used for # tracking - def run(examplegroup, reporter) + def run(example_group, reporter) @thread_array.push Thread.start { - examplegroup.run_parallel(reporter, @max_threads, @mutex, @used_threads) + example_group.run_parallel(reporter, @max_threads, @mutex, @used_threads) @thread_array.delete Thread.current } end @@ -40,8 +40,8 @@ def run(examplegroup, reporter) # remove themselves from the @thread_array so an empty array means they # completed def wait_for_completion - while @thread_array.length > 0 - sleep 1 + @thread_array.each do |t| + t.join end end end diff --git a/lib/rspec/core/example_thread_runner.rb b/lib/rspec/core/example_thread_runner.rb index f2296f7e22..7f05c33f28 100644 --- a/lib/rspec/core/example_thread_runner.rb +++ b/lib/rspec/core/example_thread_runner.rb @@ -22,7 +22,7 @@ def initialize(num_threads, used_threads) # at or over the allocated maximum it will wait until a thread is available def wait_for_available_thread while @used_threads.to_i >= @num_threads.to_i - sleep 1 + sleep 0.1 end end @@ -46,8 +46,8 @@ def run(example, instance, reporter) # remove themselves from the @thread_array so an empty array means they # completed def wait_for_completion - while @thread_array.length > 0 - sleep 1 + @thread_array.each do |t| + t.join end end end diff --git a/spec/command_line/parallel_spec.rb b/spec/command_line/parallel_spec.rb index a1478f7798..a79ba893ec 100644 --- a/spec/command_line/parallel_spec.rb +++ b/spec/command_line/parallel_spec.rb @@ -1,38 +1,36 @@ require 'spec_helper' - - RSpec.describe 'command line', :ui, :slow do before :each do @out = StringIO.new write_file 'spec/parallel_spec.rb', """ - RSpec.describe \"parallel run\" do - it \"thread 0 example\" do + RSpec.describe 'parallel run' do + it 'thread 0 example' do sleep 1 fail end - it \"thread 1 example\" do + it 'thread 1 example' do sleep 1 pass end - it \"thread 2 example\" do + it 'thread 2 example' do sleep 1 pass end - it \"thread 3 example\" do + it 'thread 3 example' do sleep 1 fail end - it \"thread 4 example\" do + it 'thread 4 example' do sleep 1 pass end - it \"thread 5 example\" do + it 'thread 5 example' do sleep 1 pass end