Abort bisect if the ordering is inconsistent.

lib/rspec/core/bisect/coordinator.rb

@@ -36,8 +36,8 @@ def bisect
 
             reporter.publish(:bisect_repro_command, :repro => repro)
           end
-        rescue Server::DidNotGetRunResults => e
-          reporter.publish(:bisect_failed, :run_output => e.run_output)
+        rescue BisectFailedError => e
+          reporter.publish(:bisect_failed, :failure_explanation => e.message)
         end
 
       private

lib/rspec/core/bisect/example_minimizer.rb

@@ -59,6 +59,8 @@ def get_same_failures?(ids)
           results = runner.run(ids + failed_example_ids)
           notify(:individual_run_complete)
 
+          abort_if_ordering_inconsistent(results)
+
           (results.failed_example_ids == failed_example_ids).tap do |same|
             if same
               debug 2, "Running with #{ids}, got same failures"
@@ -105,6 +107,15 @@ def track_duration
           [yield, ::RSpec::Core::Time.now - start]
         end
 
+        def abort_if_ordering_inconsistent(results)
+          expected_order = all_example_ids & results.all_example_ids
+          return if expected_order == results.all_example_ids
+
+          raise BisectFailedError, "\n\nThe example ordering is inconsistent. " \
+                "`--bisect` relies upon consistent ordering (e.g. by passing " \
+                "`--seed` if you're using random ordering) to work properly."
+        end
+
         def notify(*args)
           reporter.publish(*args)
         end

lib/rspec/core/bisect/server.rb

@@ -4,6 +4,9 @@ module RSpec
   module Core
     # @private
     module Bisect
+      # @private
+      BisectFailedError = Class.new(StandardError)
+
       # @private
       # A DRb server that receives run results from a separate RSpec process
       # started by the bisect process.
@@ -16,21 +19,11 @@ def self.run
           server.stop
         end
 
-        # @private
-        class DidNotGetRunResults < StandardError
-          attr_reader :run_output
-
-          def initialize(run_output)
-            @run_output = run_output
-            super("Did not get run results, but got output: #{run_output}")
-          end
-        end
-
         def capture_run_results(expected_failures=[])
           self.expected_failures  = expected_failures
           self.latest_run_results = nil
           run_output = yield
-          latest_run_results || raise(DidNotGetRunResults, run_output)
+          latest_run_results || raise_bisect_failed(run_output)
         end
 
         def start
@@ -51,6 +44,13 @@ def drb_port
 
         # Set via DRb by the BisectFormatter with the results of the run.
         attr_accessor :latest_run_results
+
+      private
+
+        def raise_bisect_failed(run_output)
+          raise BisectFailedError, "Failed to get results from the spec " \
+                "run. Spec run output:\n\n#{run_output}"
+        end
       end
     end
   end

lib/rspec/core/formatters/bisect_progress_formatter.rb

@@ -54,10 +54,7 @@ def bisect_repro_command(notification)
         end
 
         def bisect_failed(notification)
-          output.puts
-          output.puts ConsoleCodes.wrap("Spec run failed!", :failure)
-          output.puts
-          output.puts ConsoleCodes.wrap(notification.run_output, :failure)
+          output.puts "\nBisect failed! #{notification.failure_explanation}"
         end
       end
     end

spec/integration/bisect_spec.rb

@@ -4,7 +4,7 @@ module RSpec::Core
 
     def bisect(cli_args)
       RSpec.configuration.output_stream = formatter_output
-      parser = Parser.new(cli_args)
+      parser = Parser.new(cli_args + ["--bisect"])
       expect(parser).to receive(:exit)
 
       expect {
@@ -15,7 +15,7 @@ def bisect(cli_args)
     end
 
     it 'finds the minimum rerun command and exits' do
-      output = bisect(%w[spec/rspec/core/resources/order_dependent_specs.rb --order defined --bisect])
+      output = bisect(%w[spec/rspec/core/resources/order_dependent_specs.rb --order defined])
 
       expect(output).to eq(<<-EOS.gsub(/^\s+\|/, ''))
         |Bisect started using options: "spec/rspec/core/resources/order_dependent_specs.rb --order defined"
@@ -36,8 +36,15 @@ def bisect(cli_args)
 
     context "when a load-time problem occurs while running the suite" do
       it 'surfaces the stdout and stderr output to the user' do
-        output = bisect(%w[spec/rspec/core/resources/fail_on_load_spec.rb_ --bisect])
-        expect(output).to include("Spec run failed", "undefined method `contex'", "About to call misspelled method")
+        output = bisect(%w[spec/rspec/core/resources/fail_on_load_spec.rb_])
+        expect(output).to include("Bisect failed!", "undefined method `contex'", "About to call misspelled method")
+      end
+    end
+
+    context "when the spec ordering is inconsistent" do
+      it 'stops bisecting and surfaces the problem to the user' do
+        output = bisect(%W[spec/rspec/core/resources/inconsistently_ordered_specs.rb])
+        expect(output).to include("Bisect failed!", "The example ordering is inconsistent")
       end
     end
   end

spec/rspec/core/bisect/example_minimizer_spec.rb

@@ -11,7 +11,8 @@ def original_cli_args
       end
 
       def original_results
-        RunResults.new(all_ids, always_failures | dependent_failures.keys)
+        failures = always_failures | dependent_failures.keys
+        RunResults.new(all_ids, failures.sort)
       end
 
       def run(ids)
@@ -20,7 +21,7 @@ def run(ids)
           failures << failing_example if ids.include?(depends_upon)
         end
 
-        RunResults.new(ids, failures)
+        RunResults.new(ids.sort, failures.sort)
       end
     end
 

spec/rspec/core/resources/inconsistently_ordered_specs.rb

@@ -0,0 +1,12 @@
+# Deliberately named _specs.rb to avoid being loaded except when specified
+
+RSpec.configure do |c|
+  c.register_ordering(:global, &:shuffle)
+end
+
+10.times do |i|
+  RSpec.describe "Group #{i}" do
+    it("passes") {      }
+    it("fails")  { fail }
+  end
+end