Skip to content

Commit

Permalink
Improve order-dependency check debug output
Browse files Browse the repository at this point in the history
These messages previously ran on the same line as a test run, which made for
confusing output.
  • Loading branch information
urbanautomaton committed Jun 26, 2015
1 parent 441fea5 commit 1d5dc50
Show file tree
Hide file tree
Showing 3 changed files with 62 additions and 24 deletions.
4 changes: 2 additions & 2 deletions features/command_line/bisect.feature
Original file line number Diff line number Diff line change
Expand Up @@ -108,8 +108,8 @@ Feature: Bisect
- ./spec/calculator_8_spec.rb[1:1]
- ./spec/calculator_9_spec.rb[1:1]
Checking that failure(s) are order-dependent..
- Running: rspec ./spec/calculator_1_spec.rb[1:1] --seed 1234 (n.nnnn seconds) failure appears to be order-dependent
- Running: rspec ./spec/calculator_1_spec.rb[1:1] --seed 1234 (n.nnnn seconds)
- Failure appears to be order-dependent
Round 1: bisecting over non-failing examples 1-9
- Running: rspec ./spec/calculator_1_spec.rb[1:1] ./spec/calculator_6_spec.rb[1:1] ./spec/calculator_7_spec.rb[1:1] ./spec/calculator_8_spec.rb[1:1] ./spec/calculator_9_spec.rb[1:1] --seed 1234 (0.15302 seconds)
- Running: rspec ./spec/calculator_10_spec.rb[1:1] ./spec/calculator_1_spec.rb[1:1] ./spec/calculator_2_spec.rb[1:1] ./spec/calculator_3_spec.rb[1:1] ./spec/calculator_4_spec.rb[1:1] ./spec/calculator_5_spec.rb[1:1] --seed 1234 (0.19708 seconds)
Expand Down
9 changes: 8 additions & 1 deletion lib/rspec/core/formatters/bisect_progress_formatter.rb
Original file line number Diff line number Diff line change
Expand Up @@ -84,7 +84,6 @@ def bisect_aborted(notification)
output.puts "\n\nBisect aborted!"
output.puts "\nThe most minimal reproduction command discovered so far is:\n #{notification.repro}"
end

end

# @private
Expand All @@ -110,6 +109,14 @@ def bisect_individual_run_complete(notification)
output.print " (#{Helpers.format_duration(notification.duration)})"
end

def bisect_dependency_check_passed(_notification)
output.print "\n - Failure appears to be order-dependent"
end

def bisect_dependency_check_failed(_notification)
output.print "\n - Failure is not order-dependent"
end

def bisect_round_started(notification)
super(notification, false)
end
Expand Down
73 changes: 52 additions & 21 deletions spec/rspec/core/bisect/coordinator_spec.rb
Original file line number Diff line number Diff line change
Expand Up @@ -46,25 +46,6 @@ def find_minimal_repro(output, formatter=Formatters::BisectProgressFormatter)
EOS
end

it 'detects non-order-dependent failures and prints the minimal reproduction' do
fake_runner.dependent_failures = {}
output = StringIO.new
find_minimal_repro(output)
output = normalize_durations(output.string)

expect(output).to eq(<<-EOS.gsub(/^\s+\|/, ''))
|Bisect started using options: ""
|Running suite to find failures... (n.nnnn seconds)
|Starting bisect with 1 failing example and 7 non-failing examples.
|Checking that failure(s) are order-dependent... failure is not order-dependent
|
|Bisect complete! Reduced necessary non-failing examples from 7 to 0 in n.nnnn seconds.
|
|The minimal reproduction command is:
| rspec 2.rb[1:1]
EOS
end

it 'can use the bisect debug formatter to get detailed progress' do
output = StringIO.new
find_minimal_repro(output, Formatters::BisectDebugFormatter)
Expand All @@ -84,8 +65,8 @@ def find_minimal_repro(output, formatter=Formatters::BisectProgressFormatter)
| - 7.rb[1:1]
| - 8.rb[1:1]
|Checking that failure(s) are order-dependent..
| - Running: rspec 2.rb[1:1] 5.rb[1:1] (n.nnnn seconds) failure appears to be order-dependent
|
| - Running: rspec 2.rb[1:1] 5.rb[1:1] (n.nnnn seconds)
| - Failure appears to be order-dependent
|Round 1: bisecting over non-failing examples 1-6
| - Running: rspec 2.rb[1:1] 5.rb[1:1] 6.rb[1:1] 7.rb[1:1] 8.rb[1:1] (n.nnnn seconds)
| - Running: rspec 1.rb[1:1] 2.rb[1:1] 3.rb[1:1] 4.rb[1:1] 5.rb[1:1] (n.nnnn seconds)
Expand Down Expand Up @@ -116,6 +97,56 @@ def find_minimal_repro(output, formatter=Formatters::BisectProgressFormatter)
EOS
end

context "with an order-independent failure" do
it "detects the independent case and prints the minimal reproduction" do
fake_runner.dependent_failures = {}
output = StringIO.new
find_minimal_repro(output)
output = normalize_durations(output.string)

expect(output).to eq(<<-EOS.gsub(/^\s+\|/, ''))
|Bisect started using options: ""
|Running suite to find failures... (n.nnnn seconds)
|Starting bisect with 1 failing example and 7 non-failing examples.
|Checking that failure(s) are order-dependent... failure is not order-dependent
|
|Bisect complete! Reduced necessary non-failing examples from 7 to 0 in n.nnnn seconds.
|
|The minimal reproduction command is:
| rspec 2.rb[1:1]
EOS
end

it "can use the debug formatter for detailed output" do
fake_runner.dependent_failures = {}
output = StringIO.new
find_minimal_repro(output, Formatters::BisectDebugFormatter)
output = normalize_durations(output.string)

expect(output).to eq(<<-EOS.gsub(/^\s+\|/, ''))
|Bisect started using options: ""
|Running suite to find failures... (n.nnnn seconds)
| - Failing examples (1):
| - 2.rb[1:1]
| - Non-failing examples (7):
| - 1.rb[1:1]
| - 3.rb[1:1]
| - 4.rb[1:1]
| - 5.rb[1:1]
| - 6.rb[1:1]
| - 7.rb[1:1]
| - 8.rb[1:1]
|Checking that failure(s) are order-dependent..
| - Running: rspec 2.rb[1:1] (n.nnnn seconds)
| - Failure is not order-dependent
|Bisect complete! Reduced necessary non-failing examples from 7 to 0 in n.nnnn seconds.
|
|The minimal reproduction command is:
| rspec 2.rb[1:1]
EOS
end
end

context "when the user aborst the bisect with ctrl-c" do
let(:aborting_formatter) do
Class.new(Formatters::BisectProgressFormatter) do
Expand Down

0 comments on commit 1d5dc50

Please sign in to comment.