allow explicit runtime ordering

Gemfile

@@ -3,7 +3,7 @@ gemspec
 
 gem 'bump'
 gem 'test-unit'
-gem 'minitest', '~> 4.7.5'
+gem 'minitest', '~> 5.5.0'
 gem 'rspec', '>=2.4'
 gem 'cucumber'
 gem 'spinach'

Gemfile.lock

@@ -22,7 +22,7 @@ GEM
     gherkin (2.12.2-java)
       multi_json (~> 1.3)
     gherkin-ruby (0.3.0)
-    minitest (4.7.5)
+    minitest (5.5.1)
     multi_json (1.8.2)
     multi_test (0.0.2)
     parallel (1.3.3)
@@ -47,7 +47,7 @@ PLATFORMS
 DEPENDENCIES
   bump
   cucumber
-  minitest (~> 4.7.5)
+  minitest (~> 5.5.0)
   parallel_tests!
   rake
   rspec (>= 2.4)

Readme.md

@@ -99,7 +99,7 @@ Rspec: Add to your `.rspec_parallel` (or `.rspec`) :
     --format progress
     --format ParallelTests::RSpec::RuntimeLogger --out tmp/parallel_runtime_rspec.log
 
-### Test::Unit & Minitest 4
+### Test::Unit & Minitest 4/5
 
 Add to your `test_helper.rb`:
 ```ruby
@@ -175,7 +175,8 @@ Options are:
           steps - number of cucumber/spinach steps
           scenarios - individual cucumber scenarios
           filesize - by size of the file
-          default - runtime or filesize
+          runtime - info from runtime log
+          default - runtime when runtime log is filled otherwise filesize
     -m, --multiply-processes [FLOAT] use given number as a multiplier of processes to run
     -s, --single [PATTERN]           Run all matching files in the same process
     -i, --isolate                    Do not run any other tests in the group used by --single(-s)

lib/parallel_tests/cli.rb

@@ -108,7 +108,8 @@ def parse_options!(argv)
                     steps - number of cucumber/spinach steps
                     scenarios - individual cucumber scenarios
                     filesize - by size of the file
-                    default - runtime or filesize
+                    runtime - info from runtime log
+                    default - runtime when runtime log is filled otherwise filesize
           TEXT
           ) { |type| options[:group_by] = type.to_sym }
         opts.on("-m [FLOAT]", "--multiply-processes [FLOAT]", Float, "use given number as a multiplier of processes to run") { |multiply| options[:multiply] = multiply }
@@ -162,7 +163,10 @@ def parse_options!(argv)
       options[:group_by] ||= :filesize if options[:only_group]
 
       raise "--group-by found and --single-process are not supported" if options[:group_by] == :found and options[:single_process]
-      raise "--group-by filesize is required for --only-group" if options[:group_by] != :filesize and options[:only_group]
+      allowed = [:filesize, :runtime, :found]
+      if !allowed.include?(options[:group_by]) && options[:only_group]
+        raise "--group-by #{allowed.join(" or ")} is required for --only-group"
+      end
 
       options
     end

lib/parallel_tests/test/runner.rb

@@ -41,13 +41,26 @@ def line_is_result?(line)
         def tests_in_groups(tests, num_groups, options={})
           tests = find_tests(tests, options)
 
-          tests = if options[:group_by] == :found
-            tests.map { |t| [t, 1] }
-          elsif options[:group_by] == :filesize
-            with_filesize_info(tests)
+          case options[:group_by]
+          when :found
+            tests.map! { |t| [t, 1] }
+          when :filesize
+            sort_by_filesize(tests)
+          when :runtime
+            sort_by_runtime(tests, runtimes(options))
+          when nil
+            # use recorded test runtime if we got enough data
+            runtimes = runtimes(options) rescue []
+            if runtimes.size * 1.5 > tests.size
+              puts "Using recorded test runtime"
+              sort_by_runtime(tests, runtimes)
+            else
+              sort_by_filesize(tests)
+            end
           else
-            with_runtime_info(tests, options)
+            raise ArgumentError, "Unsupported option #{options[:group_by]}"
           end
+
           Grouper.in_even_groups_by_size(tests, num_groups, options)
         end
 
@@ -136,28 +149,25 @@ def capture_output(out, silence)
           result
         end
 
-        def with_runtime_info(tests, options = {})
-          log = options[:runtime_log] || runtime_log
-          lines = File.read(log).split("\n") rescue []
-
-          # use recorded test runtime if we got enough data
-          if lines.size * 1.5 > tests.size
-            puts "Using recorded test runtime: #{log}"
-            times = Hash.new(1)
-            lines.each do |line|
-              test, time = line.split(":")
-              next unless test and time
-              times[test] = time.to_f
-            end
-            tests.sort.map { |test| [test, times[test]] }
-          else # use file sizes
-            with_filesize_info(tests)
+        def sort_by_runtime(tests, runtimes)
+          times = {}
+          runtimes.each do |line|
+            test, time = line.split(":", 2)
+            next unless test and time
+            times[test] = time.to_f
           end
+          tests.sort!
+          tests.map! { |test| [test, times[test] || 1] }
+        end
+
+        def runtimes(options)
+          log = options[:runtime_log] || runtime_log
+          File.read(log).split("\n")
         end
 
-        def with_filesize_info(tests)
-          # use filesize to group files
-          tests.sort.map { |test| [test, File.stat(test).size] }
+        def sort_by_filesize(tests)
+          tests.sort!
+          tests.map! { |test| [test, File.stat(test).size] }
         end
 
         def find_tests(tests, options = {})

lib/parallel_tests/test/runtime_logger.rb

@@ -71,7 +71,25 @@ def logfile
   end
 end
 
-if defined?(MiniTest::Unit)
+if defined?(Minitest::Test) # Minitest 5
+  class << Minitest::Runnable
+    alias_method :run_without_runtime_log, :run
+    def run(*args)
+      ParallelTests::Test::RuntimeLogger.log_test_run(self) do
+        run_without_runtime_log(*args)
+      end
+    end
+  end
+
+  class << Minitest
+    alias_method :run_without_runtime_log, :run
+    def run(*args)
+      result = run_without_runtime_log(*args)
+      ParallelTests::Test::RuntimeLogger.unique_log
+      result
+    end
+  end
+elsif defined?(MiniTest::Unit) # Minitest 4
   MiniTest::Unit.class_eval do
     alias_method :_run_suite_without_runtime_log, :_run_suite
     def _run_suite(*args)
@@ -87,7 +105,7 @@ def _run_suites(*args)
       result
     end
   end
-else
+else # Test::Unit
   require 'test/unit/testsuite'
   class ::Test::Unit::TestSuite
     alias_method :run_without_timing, :run

spec/fixtures/minitest4/Gemfile

@@ -0,0 +1,4 @@
+source "https://rubygems.org"
+
+gemspec path: "../../../"
+gem "minitest", "~> 4.7.5"

spec/fixtures/minitest4/Gemfile.lock

@@ -0,0 +1,18 @@
+PATH
+  remote: ../../../
+  specs:
+    parallel_tests (1.3.1)
+      parallel
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    minitest (4.7.5)
+    parallel (1.4.0)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  minitest (~> 4.7.5)
+  parallel_tests!

spec/fixtures/minitest4/test/0_test.rb

@@ -0,0 +1,17 @@
+require 'bundler/setup'
+require 'minitest/autorun'
+require 'parallel_tests/test/runtime_logger'
+
+class Foo0 < MiniTest::Unit::TestCase
+  def test_foo
+    sleep 0.5
+    assert true
+  end
+end
+
+class Bar0 < MiniTest::Unit::TestCase
+  def test_foo
+    sleep 0.25
+    assert true
+  end
+end

spec/fixtures/minitest4/test/1_test.rb

@@ -0,0 +1,17 @@
+require 'bundler/setup'
+require 'minitest/autorun'
+require 'parallel_tests/test/runtime_logger'
+
+class Foo1 < MiniTest::Unit::TestCase
+  def test_foo
+    sleep 0.5
+    assert true
+  end
+end
+
+class Bar1 < MiniTest::Unit::TestCase
+  def test_foo
+    sleep 0.25
+    assert true
+  end
+end

spec/parallel_tests/cli_spec.rb

@@ -48,7 +48,11 @@ def call(*args)
 
     context "parse only-group" do
       it "group_by should be set to filesize" do
-        call(["test", "--only-group", '1']).should == defaults.merge(:group_by=>:filesize, :only_group => [1])
+        call(["test", "--only-group", '1']).should == defaults.merge(only_group: [1], group_by: :filesize)
+      end
+
+      it "allows runtime" do
+        call(["test", "--only-group", '1', '--group-by', 'runtime']).should == defaults.merge(only_group: [1], group_by: :runtime)
       end
 
       it "raise error when group_by isn't filesize" do
@@ -57,16 +61,12 @@ def call(*args)
         }.to raise_error(RuntimeError)
       end
 
-      context "with group_by default to filesize" do
-        let(:defaults_with_filesize){defaults.merge(:group_by => :filesize)}
-
-        it "with multiple groups" do
-          call(["test", "--only-group", '4,5']).should == defaults_with_filesize.merge(:only_group => [4,5])
-        end
+      it "with multiple groups" do
+        call(["test", "--only-group", '4,5']).should == defaults.merge(only_group: [4,5], group_by: :filesize)
+      end
 
-        it "with a single group" do
-          call(["test", "--only-group", '4']).should == defaults_with_filesize.merge(:only_group => [4])
-        end
+      it "with a single group" do
+        call(["test", "--only-group", '4']).should == defaults.merge(:only_group => [4], group_by: :filesize)
       end
     end
   end

spec/parallel_tests/test/runner_spec.rb

@@ -32,43 +32,53 @@ def call(*args)
       ParallelTests::Test::Runner.tests_in_groups(*args)
     end
 
-    it "does not sort when passed false do_sort option" do
-      ParallelTests::Test::Runner.should_not_receive(:smallest_first)
-      call([], 1, :group_by => :found)
+    it "raises when passed invalid group" do
+      expect { call([], 1, group_by: :sdjhfdfdjs) }.to raise_error(ArgumentError)
     end
 
-    it "does sort when not passed do_sort option" do
-      ParallelTests::Test::Runner.stub!(:tests_with_runtime).and_return([])
-      ParallelTests::Grouper.should_receive(:group_features_by_size).and_return([])
-      call([], 1)
+    it "uses given when passed found" do
+      call(["a", "b", "c"], 2, group_by: :found).should == [["a", "c"], ["b"]]
     end
 
-    it "groups by single_process pattern and then via size" do
-      ParallelTests::Test::Runner.should_receive(:with_runtime_info).
-        and_return([
-          ['aaa', 5],
-          ['aaa2', 5],
-          ['bbb', 2],
-          ['ccc', 1],
-          ['ddd', 1]
-        ])
-      result = call([], 3, :single_process => [/^a.a/])
-      result.should == [["aaa", "aaa2"], ["bbb"], ["ccc", "ddd"]]
+    context "when passed no group" do
+      it "sort by file size" do
+        File.should_receive(:stat).with("a").and_return 1
+        File.should_receive(:stat).with("b").and_return 1
+        File.should_receive(:stat).with("c").and_return 3
+        call(["a", "b", "c"], 2)
+      end
+
+      it "sorts by runtime when runtime is available" do
+        ParallelTests::Test::Runner.should_receive(:puts).with("Using recorded test runtime")
+        ParallelTests::Test::Runner.should_receive(:runtimes).and_return(["a:1", "b:1", "c:3"])
+        call(["a", "b", "c"], 2).should == [["c"], ["a", "b"]]
+      end
+
+      it "sorts by filesize when runtime is too little" do
+        ParallelTests::Test::Runner.should_not_receive(:puts)
+        ParallelTests::Test::Runner.should_receive(:runtimes).and_return(["a:1"])
+        File.should_receive(:stat).with("a").and_return 1
+        File.should_receive(:stat).with("b").and_return 1
+        File.should_receive(:stat).with("c").and_return 3
+        call(["a", "b", "c"], 2)
+      end
     end
 
-    it "groups by size and adds isolated separately" do
-      pending if RUBY_PLATFORM == "java"
-      ParallelTests::Test::Runner.should_receive(:with_runtime_info).
-        and_return([
-          ['aaa', 0],
-          ['bbb', 3],
-          ['ccc', 1],
-          ['ddd', 2],
-          ['eee', 1]
-        ])
-
-      result = call([], 3, :isolate => true, :single_process => [/^aaa/])
-      result.should == [["aaa"], ["bbb", "eee"], ["ccc", "ddd"]]
+    context "when passed runtime" do
+      it "groups by single_process pattern and then via size" do
+        ParallelTests::Test::Runner.should_receive(:runtimes).
+          and_return(%w[aaa:5 aaa2:5 bbb:2 ccc:1 ddd:1])
+        result = call(["aaa", "aaa2", "bbb", "ccc", "ddd"], 3, single_process: [/^a.a/], group_by: :runtime)
+        result.should == [["aaa", "aaa2"], ["bbb"], ["ccc", "ddd"]]
+      end
+
+      it "groups by size and adds isolated separately" do
+        pending if RUBY_PLATFORM == "java"
+        ParallelTests::Test::Runner.should_receive(:runtimes).
+          and_return(%w[aaa:0 bbb:3 ccc:1 ddd:2 eee:1])
+        result = call(["aaa", "bbb", "ccc", "ddd", "eee"], 3, isolate: true, single_process: [/^aaa/], group_by: :runtime)
+        result.should == [["aaa"], ["bbb", "eee"], ["ccc", "ddd"]]
+      end
     end
   end