A more minimal approach to retrying failed examples #596

Closed
wants to merge 7 commits into
from
View
15 features/command_line/example_name_option.feature
@@ -9,6 +9,8 @@ Feature: --example option
This allows you to run a single uniquely named example, all examples with
similar names, all the examples in a uniquely named group, etc, etc.
+ You can also use the option more than once to specify multiple example matches.
+
Background:
Given a file named "first_spec.rb" with:
"""
@@ -84,3 +86,16 @@ Feature: --example option
Scenario: Object#method
When I run `rspec . --example 'Array#length'`
Then the examples should all pass
+
+ Scenario: Multiple applications of example name option
+ When I run `rspec . --example 'first group' --example 'second group' --format d`
+ Then the examples should all pass
+ And the output should contain all of these:
+ |first example in first group|
+ |second example in first group|
+ |first example in second group|
+ |second example in second group|
+ And the output should not contain any of these:
+ |first example in third group|
+ |nested group first example in nested group|
+ |nested group second example in nested group|
View
64 features/command_line/rerun_failed_examples.feature
@@ -0,0 +1,64 @@
+Feature: Rerun failed examples only
+
+ Using a combination of the failures-only formatter and the -O option to
+ read options from a file, we can get rspec to retry failed examples.
+
+ Scenario: record failed examples in file
+ Given a file named "spec/example_spec.rb" with:
+ """
+ describe "retry" do
+ it "should not rerun this one" do
+ true.should == true
+ end
+ it "should rerun this one" do
+ false.should == true
+ end
+ it "should also rerun this one" do
+ false.should == true
+ end
+ end
+ """
+ When I run `rspec ./spec/example_spec.rb --format f -o failures.txt --format d`
+ Then the exit status should be 1
+ And the output should contain "3 examples, 2 failures"
+ And the output should not contain:
+ """
+ -e 'retry should rerun this one'
+ -e 'retry should also rerun this one'
+ """
+ And the file "failures.txt" should contain:
+ """
+ -e 'retry should rerun this one'
+ -e 'retry should also rerun this one'
+ """
+
+ Scenario: with examples recorded, only rerun failed examples
+ Given a file named "spec/example_spec.rb" with:
+ """
+ describe "retry" do
+ it "should not rerun this one" do
+ true.should == true
+ end
+ it "should rerun this one" do
+ false.should == true
+ end
+ it "should also rerun this one" do
+ false.should == true
+ end
+ end
+ """
+ And a file named "failures.txt" with:
+ """
+ -e 'retry should rerun this one'
+ -e 'retry should also rerun this one'
+ """
+ When I run `rspec ./spec/example_spec.rb -O failures.txt --format d`
+ Then the exit status should be 1
+ And the output should contain all of these:
+ |should rerun this one|
+ |should also rerun this one|
+ |2 failures|
+ And the output should not contain any of these:
+ |should not rerun this one|
+
+
View
39 features/formatters/failures_formatter.feature
@@ -0,0 +1,39 @@
+Feature: failures-only formatter
+
+ The failures-only formatter outputs a format suitable for feeding back to
+ RSpec itself to have it retry failed examples. It uses the -e format to
+ name the examples which have failed.
+
+ Scenario: Formatting example names for retry
+ Given a file named "failing_spec.rb" with:
+ """
+ describe "Failing" do
+ it "is guaranteed to fail here" do
+ "fail".should eq("succeed")
+ end
+
+ it "is also guaranteed to fail here" do
+ "fail".should eq("joy")
+ end
+
+ it "should fail with a 'quoted' thing" do
+ "bad".should eq("worse")
+ end
+ end
+ """
+ And a file named "passing_spec.rb" with:
+ """
+ describe "Passing" do
+ it "will pass here" do
+ true.should eq(true)
+ end
+ end
+ """
+ When I run `rspec passing_spec.rb failing_spec.rb --format f`
+ Then the output should contain all of these:
+ |-e 'Failing is guaranteed to fail here'|
+ |-e 'Failing is also guaranteed to fail here'|
+ |-e 'Failing should fail with a \\'quoted\\' thing'|
+ And the output should not contain any of these:
+ |Passing will pass here|
+ And the exit status should be 1
View
9 lib/rspec/core/configuration.rb
@@ -463,7 +463,11 @@ def line_numbers=(line_numbers)
end
def full_description=(description)
- filter_run :full_description => /#{description}/
+ if description.is_a?(Array)
+ filter_run :full_description => Regexp.union(description.map { |d| /#{d}/ })
+ else
+ filter_run :full_description => /#{description}/
+ end
end
# @overload add_formatter(formatter)
@@ -877,6 +881,9 @@ def built_in_formatter(key)
when 'p', 'progress'
require 'rspec/core/formatters/progress_formatter'
RSpec::Core::Formatters::ProgressFormatter
+ when 'f', 'failures'
+ require 'rspec/core/formatters/failures_formatter'
+ RSpec::Core::Formatters::FailuresFormatter
end
end
View
3 lib/rspec/core/drb_options.rb
@@ -42,7 +42,8 @@ def add_full_description(argv)
# into a regexp when received for the first time (see OptionParser).
# Hence, merely grabbing the source of this regexp will retain the
# backslashes, so we must remove them.
- argv << "--example" << @submitted_options[:full_description].source.delete('\\')
+ argv << @submitted_options[:full_description].map {|d| ["--example", d.source.delete('\\')] }
+ argv.flatten!
end
end
View
19 lib/rspec/core/formatters/failures_formatter.rb
@@ -0,0 +1,19 @@
+require 'rspec/core/formatters/base_formatter'
+
+module RSpec
+ module Core
+ module Formatters
+ class FailuresFormatter < BaseFormatter
+
+ def example_failed(example)
+ output.puts retry_command(example)
+ end
+
+ def retry_command(example)
+ example_name = example.full_description.gsub(%q{'}) { |c| %q{\'} }
+ "-e '#{example_name}'"
+ end
+ end
+ end
+ end
+end
View
4 lib/rspec/core/option_parser.rb
@@ -92,6 +92,7 @@ def parser(options)
' [d]ocumentation (group and example names)',
' [h]tml',
' [t]extmate',
+ ' [f]ailed example names (for use with -O)',
' custom formatter class name') do |o|
options[:formatters] ||= []
options[:formatters] << [o]
@@ -135,7 +136,8 @@ def parser(options)
end
parser.on('-e', '--example STRING', "Run examples whose full nested names include STRING") do |o|
- options[:full_description] = Regexp.compile(Regexp.escape(o))
+ options[:full_description] ||= []
+ options[:full_description] << Regexp.compile(Regexp.escape(o))
end
parser.on('-l', '--line_number LINE', 'Specify line number of an example or group (may be',
View
8 spec/rspec/core/configuration_options_spec.rb
@@ -170,8 +170,8 @@
describe "--example" do
it "sets :full_description" do
- parse_options('--example','foo').should include(:full_description => /foo/)
- parse_options('-e','bar').should include(:full_description => /bar/)
+ parse_options('--example','foo').should include(:full_description => [/foo/])
+ parse_options('-e','bar').should include(:full_description => [/bar/])
end
end
@@ -321,7 +321,7 @@
options[:color].should be_true
options[:line_numbers].should eq(["37"])
options[:debug].should be_true
- options[:full_description].should eq(/foo\ bar/)
+ options[:full_description].should eq([/foo\ bar/])
options[:drb].should be_true
end
@@ -354,7 +354,7 @@
it "parses -e 'full spec description'" do
File.open("./custom.opts", "w") {|f| f << "-e 'The quick brown fox jumps over the lazy dog'"}
options = parse_options("-O", "./custom.opts")
- options[:full_description].should == /The\ quick\ brown\ fox\ jumps\ over\ the\ lazy\ dog/
+ options[:full_description].should == [/The\ quick\ brown\ fox\ jumps\ over\ the\ lazy\ dog/]
end
end
end
View
5 spec/rspec/core/configuration_spec.rb
@@ -407,6 +407,11 @@ module RSpec::Core
config.filter.should eq({:full_description => /foo/})
end
+ it "assigns the example names as the filter on description if description is an array" do
+ config.full_description = [ "foo", "bar" ]
+ config.filter.should eq({:full_description => Regexp.union(/foo/, /bar/)})
+ end
+
describe "#default_path" do
it 'defaults to "spec"' do
config.default_path.should eq('spec')
View
2 spec/rspec/core/option_parser_spec.rb
@@ -68,7 +68,7 @@ module RSpec::Core
describe option do
it "escapes the arg" do
options = Parser.parse!([option, "this (and that)"])
- "this (and that)".should match(options[:full_description])
+ "this (and that)".should match(options[:full_description].first)
end
end
end