Merge 95fc65a into df5b27e

Rakefile

@@ -11,7 +11,7 @@ end
 
 desc 'Run spinach features'
 task :spinach do
-  exec "bin/spinach"
+  exec "bin/spinach --rand"
 end
 
 task :default => [:test, :spinach]

features/randomization.feature

@@ -0,0 +1,16 @@
+Feature: Randomizing Features & Scenarios
+  In order to ensure my tests aren't dependent
+  As a developer
+  I want spinach to randomize features and scenarios (but not steps)
+
+  Scenario: Randomizing the run without specifying a seed
+    Given I have 2 features with 2 scenarios each
+    When I randomize the run without specifying a seed
+    Then The features and scenarios are run
+    And The runner output shows a seed
+
+  Scenario: Specifying the seed
+    Given I have 2 features with 2 scenarios each
+    When I specify the seed for the run
+    Then The features and scenarios are run in a different order
+    And The runner output shows the seed

features/reporting/display_run_summary.feature

@@ -1,9 +1,25 @@
 Feature: Display run summary
   As a developer
   I want spinach to display a summary of steps statuses
-  So I can easyly know general features status
+  So I can easily know general features status
 
-  Scenario: Display run summary at the end of features run
+  Scenario: Display run summary at the end of features run without randomization
     Given I have a feature that has some successful, undefined, failed and error steps
-    When I run it
+
+    When I run it without randomization
+    Then I should see a summary with steps status information
+    And I shouldn't see a randomization seed
+
+  Scenario: Display run summary at the end of features run with randomization
+    Given I have a feature that has some successful, undefined, failed and error steps
+
+    When I run it with randomization
+    Then I should see a summary with steps status information
+    And I should see a randomization seed
+
+  Scenario: Display run summary at the end of features run with a randomization seed
+    Given I have a feature that has some successful, undefined, failed and error steps
+
+    When I run it with a specific randomization seed
     Then I should see a summary with steps status information
+    And I should see that specific randomization seed

features/steps/randomizing_features_scenarios.rb

@@ -0,0 +1,68 @@
+class Spinach::Features::RandomizingFeaturesScenarios < Spinach::FeatureSteps
+  include Integration::SpinachRunner
+
+  step 'I have 2 features with 2 scenarios each' do
+    write_file 'features/success_a.feature', <<-FEATURE
+Feature: Success A
+  Scenario: A1
+    Then a1
+  Scenario: A2
+    Then a2
+    FEATURE
+
+    write_file 'features/steps/success_a.rb', <<-STEPS
+class Spinach::Features::SuccessA < Spinach::FeatureSteps
+  step 'a1' do; end
+  step 'a2' do; end
+end
+    STEPS
+
+    write_file 'features/success_b.feature', <<-FEATURE
+Feature: Success B
+  Scenario: B1
+    Then b1
+  Scenario: B2
+    Then b2
+    FEATURE
+
+    write_file 'features/steps/success_b.rb', <<-STEPS
+class Spinach::Features::SuccessB < Spinach::FeatureSteps
+  step 'b1' do; end
+  step 'b2' do; end
+end
+    STEPS
+  end
+
+  step 'I randomize the run without specifying a seed' do
+    run_spinach({append: "--rand"})
+  end
+
+  step 'I specify the seed for the run' do
+    # Reverse order (A2 A1 B2 B1) is the only way I can show that
+    # scenarios and features are randomized by the seed when the
+    # example has 2 features each with 2 scenarios. I tried seeds
+    # until I found one that ordered the test in that order.
+    @seed = 1
+
+    run_spinach({append: "--seed #{@seed}"})
+  end
+
+  step 'The features and scenarios are run' do
+    @stdout.must_include("A1")
+    @stdout.must_include("A2")
+    @stdout.must_include("B1")
+    @stdout.must_include("B2")
+  end
+
+  step 'The features and scenarios are run in a different order' do
+    @stdout.must_match(/B2.*B1.*A2.*A1/m)
+  end
+
+  step 'The runner output shows a seed' do
+    @stdout.must_match(/^Randomized with seed \d*$/)
+  end
+
+  step 'The runner output shows the seed' do
+    @stdout.must_match(/^Randomized with seed #{@seed}$/)
+  end
+end

features/steps/reporting/display_run_summary.rb

@@ -62,7 +62,7 @@ class DisplayRunSummary < Spinach::FeatureSteps
     @feature = "features/test_feature.feature"
   end
 
-  When "I run it" do
+  When "I run it without randomization" do
     run_feature @feature
   end
 
@@ -71,4 +71,32 @@ class DisplayRunSummary < Spinach::FeatureSteps
       /Summary:.*4.*Successful.*1.*Undefined.*1.*Failed.*1.*Error/
     )
   end
+
+  And "I shouldn't see a randomization seed" do
+    @stdout.wont_match(
+      /Randomized\ with\ seed\ \d+/
+    )
+  end
+
+  When "I run it with randomization" do
+    run_feature @feature, {append: "--rand"}
+  end
+
+  And "I should see a randomization seed" do
+    @stdout.must_match(
+      /Randomized\ with\ seed\ \d+/
+    )
+  end
+
+  When "I run it with a specific randomization seed" do
+    @seed = rand(0xFFFF)
+
+    run_feature @feature, {append: "--seed #{@seed}"}
+  end
+
+  And "I should see that specific randomization seed" do
+    @stdout.must_match(
+      /Randomized\ with\ seed\ #{@seed}/
+    )
+  end
 end

features/support/spinach_runner.rb

@@ -23,6 +23,17 @@ def run_feature(feature, options={})
       run "#{ruby} #{spinach} #{feature} #{options[:append]}", options[:env]
     end
 
+    def run_spinach(options = {})
+      options[:framework] ||= :minitest
+
+      use_minitest if options[:framework] == :minitest
+      use_rspec    if options[:framework] == :rspec
+
+      spinach = File.expand_path("bin/spinach")
+
+      run "#{ruby} #{spinach} #{options[:append]}"
+    end
+
     def ruby
       return @ruby if defined?(@ruby)
 

lib/spinach.rb

@@ -9,6 +9,7 @@
 require_relative 'spinach/dsl'
 require_relative 'spinach/feature_steps'
 require_relative 'spinach/reporter'
+require_relative 'spinach/orderers'
 require_relative 'spinach/cli'
 require_relative 'spinach/generators'
 require_relative 'spinach/auditor'

lib/spinach/cli.rb

@@ -131,6 +131,16 @@ def parse_options
             config[:reporter_classes] = names
           end
 
+          opts.on('--rand', "Randomize the order of features and scenarios") do
+            config[:orderer_class] = orderer_class(:random)
+          end
+
+          opts.on('--seed SEED', Integer,
+                  "Provide a seed for randomizing the order of features and scenarios") do |seed|
+            config[:orderer_class] = orderer_class(:random)
+            config[:seed]          = seed
+          end
+
           opts.on_tail('--fail-fast',
                        'Terminate the suite run on the first failure') do |class_name|
             config[:fail_fast] = true
@@ -164,7 +174,7 @@ def fail!(message=nil)
     # Builds the class name to use an output reporter.
     #
     # @param [String] klass
-    #   The class name fo the reporter.
+    #   The class name of the reporter.
     #
     # @return [String]
     #   The full name of the reporter class.
@@ -177,5 +187,22 @@ def fail!(message=nil)
     def reporter_class(klass)
       "Spinach::Reporter::" + Spinach::Support.camelize(klass)
     end
+
+    # Builds the class to use an orderer.
+    #
+    # @param [String] klass
+    #   The class name of the orderer.
+    #
+    # @return [String]
+    #   The full name of the orderer class.
+    #
+    # @example
+    #   orderer_class('random')
+    #   # => Spinach::Orderers::Random
+    #
+    # @api private
+    def orderer_class(klass)
+      "Spinach::Orderers::" + Spinach::Support.camelize(klass)
+    end
   end
 end

lib/spinach/config.rb

@@ -33,6 +33,8 @@ class Config
                 :save_and_open_page_on_failure,
                 :reporter_classes,
                 :reporter_options,
+                :orderer_class,
+                :seed,
                 :fail_fast,
                 :audit
 
@@ -66,6 +68,29 @@ def reporter_options
       @reporter_options || {}
     end
 
+    # The "orderer class" holds the orderer class name
+    # Defaults to Spinach::Orderers::Default
+    #
+    # @return [orderer object]
+    #    The orderer that responds to specific messages.
+    #
+    # @api public
+    def orderer_class
+      @orderer_class || "Spinach::Orderers::Default"
+    end
+
+    # A randomization seed. This is what spinach uses for test run
+    # randomization, so if you call `Kernel.srand(Spinach.config.seed)`
+    # in your support environment file, not only will the test run
+    # order be guaranteed to be stable under a specific seed, all
+    # the Ruby-generated random numbers produced during your test
+    # run will also be stable under that seed.
+    #
+    # @api public
+    def seed
+      @seed ||= rand(0xFFFF)
+    end
+
     # The "step definitions path" holds the place where your feature step
     # classes will be searched for. Defaults to '#{features_path}/steps'
     #

lib/spinach/feature.rb

@@ -24,6 +24,14 @@ def run_every_scenario?
       lines_to_run.empty?
     end
 
+    # Identifier used by orderers.
+    #
+    # Needs to involve the relative file path so that the ordering
+    # a seed generates is stable across both runs and machines.
+    #
+    # @api public
+    alias ordering_id filename
+
     # Run the provided code for every step
     def each_step
       scenarios.each { |scenario| scenario.steps.each { |step| yield step } }

lib/spinach/orderers.rb

@@ -0,0 +1,2 @@
+require_relative 'orderers/default'
+require_relative 'orderers/random'

lib/spinach/orderers/default.rb

@@ -0,0 +1,25 @@
+module Spinach
+  module Orderers
+    class Default
+      def initialize(**options); end
+
+      # Appends any necessary report output (by default does nothing).
+      #
+      # @param [IO] io
+      #   Output buffer for report.
+      #
+      # @api public
+      def attach_summary(io); end
+
+      # Returns a reordered version of the provided array
+      #
+      # @param [Array] items
+      #   Items to order
+      #
+      # @api public
+      def order(items)
+        items
+      end
+    end
+  end
+end

lib/spinach/orderers/random.rb

@@ -0,0 +1,35 @@
+require 'digest'
+
+module Spinach
+  module Orderers
+    class Random
+      attr_reader :seed
+
+      def initialize(seed:)
+        @seed = seed.to_s
+      end
+
+      # Output the randomization seed in the report summary.
+      #
+      # @param [IO] io
+      #   Output buffer for report.
+      #
+      # @api public
+      def attach_summary(io)
+        io.puts("Randomized with seed #{seed}\n\n")
+      end
+
+      # Returns a reordered version of the provided array
+      #
+      # @param [Array] items
+      #   Items to order
+      #
+      # @api public
+      def order(items)
+        items.sort_by do |item|
+          Digest::MD5.hexdigest(seed + item.ordering_id)
+        end
+      end
+    end
+  end
+end

lib/spinach/reporter.rb

@@ -10,6 +10,7 @@ class Reporter
     def initialize(options = {})
       @errors = []
       @options = options
+      @orderer = options[:orderer]
       @undefined_features = []
       @successful_steps = []
       @undefined_steps = []

lib/spinach/reporter/progress.rb

@@ -8,7 +8,7 @@ class Reporter
     class Progress < Reporter
       include Reporting
 
-      # Initialitzes the runner
+      # Initializes the reporter
       #
       # @param [Hash] options
       #  Sets a custom output buffer by setting options[:output]

lib/spinach/reporter/reporting.rb

@@ -221,7 +221,9 @@ def run_summary
         error_summary      = format_summary(:red,    error_steps,      'Error')
 
         out.puts "Steps Summary: #{successful_summary}, #{pending_summary}, #{undefined_summary}, #{failed_summary}, #{error_summary}\n\n"
-        out.puts "Finished in #{Time.now - @start_time} seconds" if @start_time
+        out.puts "Finished in #{Time.now - @start_time} seconds\n\n" if @start_time
+
+        @orderer.attach_summary(out) if @orderer
       end
     end
   end

lib/spinach/reporter/stdout.rb

@@ -8,7 +8,7 @@ class Reporter
     class Stdout < Reporter
       include Reporting
 
-      # Initialitzes the runner
+      # Initializes the reporter
       #
       # @param [Hash] options
       #  Sets a custom output buffer by setting options[:output]