Specify groups argument (#797)

CHANGELOG.md

@@ -2,7 +2,7 @@
 
 ## Unreleased
 
-Add changes here in your PR
+- Add support for specifying exactly how isolated processes run tests with 'specify-groups' option.
 
 ### Breaking Changes
 

Readme.md

@@ -213,6 +213,15 @@ Options are:
     -i, --isolate                    Do not run any other tests in the group used by --single(-s).
                                      Automatically turned on if --isolate-n is set above 0.
         --isolate-n                  Number of processes for isolated groups. Default to 1 when --isolate is on.
+        --specify-groups [SPECS]     Use 'specify-groups' if you want to specify multiple specs running in multiple
+                                     processes in a specific formation. Commas indicate specs in the same process,
+                                     pipes indicate specs in a new process.  Cannot use with --single, --isolate, or
+                                     --isolate-n.  Ex.
+                                     Ex.
+                                     $ parallel_tests -n 3 . --specify-groups '1_spec.rb,2_spec.rb|3_spec.rb'
+                                       Process 1 will contain 1_spec.rb and 2_spec.rb
+                                       Process 2 will contain 3_spec.rb
+                                       Process 3 will contain all other specs
         --only-group INT[,INT]
     -e, --exec [COMMAND]             execute this code parallel and with ENV['TEST_ENV_NUMBER']
     -o, --test-options '[OPTIONS]'   execute test commands with those options

lib/parallel_tests/cli.rb

@@ -208,6 +208,23 @@ def parse_options!(argv)
           "Use 'isolate'  singles with number of processes, default: 1."
         ) { |n| options[:isolate_count] = n }
 
+        gap = "\s" * 37
+        opts.on(
+          "--specify-groups [SPECS]",
+          <<~TEXT
+            Use 'specify-groups' if you want to specify multiple specs running in multiple
+            #{gap}processes in a specific formation. Commas indicate specs in the same process,
+            #{gap}pipes indicate specs in a new process. Cannot use with --single, --isolate, or
+            #{gap}--isolate-n.  Ex.
+            #{gap}$ parallel_tests -n 3 . --specify-groups '1_spec.rb,2_spec.rb|3_spec.rb'
+            #{gap}\s\sProcess 1 will contain 1_spec.rb and 2_spec.rb
+            #{gap}\s\sProcess 2 will contain 3_spec.rb
+            #{gap}\s\sProcess 3 will contain all other specs
+          TEXT
+        ) do |groups|
+          options[:specify_groups] = groups
+        end
+
         opts.on("--only-group INT[,INT]", Array) { |groups| options[:only_group] = groups.map(&:to_i) }
 
         opts.on("-e", "--exec [COMMAND]", "execute this code parallel and with ENV['TEST_ENV_NUMBER']") { |path| options[:execute] = path }
@@ -279,6 +296,11 @@ def parse_options!(argv)
         raise "--group-by #{allowed.join(" or ")} is required for --only-group"
       end
 
+      not_allowed_with_specify_groups = [:single_process, :isolate, :isolate_count]
+      if options[:specify_groups] && (options.keys & not_allowed_with_specify_groups).any?
+        raise "Can't pass --specify-groups with any of these keys: --single, --isolate, or --isolate-n"
+      end
+
       options
     end
 

lib/parallel_tests/grouper.rb

@@ -15,6 +15,7 @@ def by_scenarios(tests, num_groups, options = {})
       def in_even_groups_by_size(items, num_groups, options = {})
         groups = Array.new(num_groups) { { items: [], size: 0 } }
 
+        return specify_groups(items, num_groups, options, groups) if options[:specify_groups]
         # add all files that should run in a single process to one group
         single_process_patterns = options[:single_process] || []
 
@@ -28,6 +29,10 @@ def in_even_groups_by_size(items, num_groups, options = {})
           raise 'Number of isolated processes must be less than total the number of processes'
         end
 
+        if isolate_count >= num_groups
+          raise 'Number of isolated processes must be less than total the number of processes'
+        end
+
         if isolate_count >= 1
           # add all files that should run in a multiple isolated processes to their own groups
           group_features_by_size(items_to_group(single_items), groups[0..(isolate_count - 1)])
@@ -46,6 +51,47 @@ def in_even_groups_by_size(items, num_groups, options = {})
 
       private
 
+      def specify_groups(items, num_groups, options, groups)
+        specify_test_process_groups = options[:specify_groups].split('|')
+        if specify_test_process_groups.count > num_groups
+          raise 'Number of processes separated by pipe must be less than or equal to the total number of processes'
+        end
+
+        all_specified_tests = specify_test_process_groups.map { |group| group.split(',') }.flatten
+        specified_items_found, items = items.partition { |item, _size| all_specified_tests.include?(item) }
+
+        specified_specs_not_found = all_specified_tests - specified_items_found.map(&:first)
+        if specified_specs_not_found.any?
+          raise "Could not find #{specified_specs_not_found} from --specify-groups in the main selected files & folders"
+        end
+
+        if specify_test_process_groups.count == num_groups && items.flatten.any?
+          raise(
+            "The number of groups in --specify-groups matches the number of groups from -n but there were other specs " \
+            "found in the main selected files & folders not specified in --specify-groups. Make sure -n is larger than the " \
+            "number of processes in --specify-groups if there are other specs that need to be run. The specs that aren't run: " \
+            "#{items.map(&:first)}"
+          )
+        end
+
+        # First order the specify_groups into the main groups array
+        specify_test_process_groups.each_with_index do |specify_test_process, i|
+          groups[i] = specify_test_process.split(',')
+        end
+
+        # Return early here as we've processed the specify_groups tests and those exactly match the items passed in
+        return groups if specify_test_process_groups.count == num_groups
+
+        # Now sort the rest of the items into the main groups array
+        remaining_groups = groups[specify_test_process_groups.count..-1]
+        group_features_by_size(items_to_group(items), remaining_groups)
+        # Don't sort all the groups, only sort the ones not specified in specify_groups
+        sorted_groups = remaining_groups.map { |g| g[:items].sort }
+        groups[specify_test_process_groups.count..-1] = sorted_groups
+
+        groups
+      end
+
       def isolate_count(options)
         if options[:isolate_count] && options[:isolate_count] > 1
           options[:isolate_count]

spec/parallel_tests/cli_spec.rb

@@ -128,6 +128,16 @@ def call(*args)
       end
     end
 
+    context "specify groups" do
+      it "groups can be just one string" do
+        expect(call(["test", "--specify-groups", 'test'])).to eq(defaults.merge(specify_groups: 'test'))
+      end
+
+      it "groups can be a string separated by commas and pipes" do
+        expect(call(["test", "--specify-groups", 'test1,test2|test3'])).to eq(defaults.merge(specify_groups: 'test1,test2|test3'))
+      end
+    end
+
     context "when the -- option separator is used" do
       it "interprets arguments as files/directories" do
         expect(call(['--', 'test'])).to eq(files: ['test'])

spec/parallel_tests/grouper_spec.rb

@@ -75,6 +75,54 @@ def call(num_groups, options = {})
         "Number of isolated processes must be less than total the number of processes"
       )
     end
+
+    it "groups specify_groups as specified when specify_groups is just one spec" do
+      expect(call(3, specify_groups: '1')).to eq([["1"], ["2", "5"], ["3", "4"]])
+    end
+
+    it "groups specify_groups as specified when specify_groups is just multiple specs in one process" do
+      expect(call(3, specify_groups: '3,1')).to eq([["3", "1"], ["5"], ["2", "4"]])
+    end
+
+    it "groups specify_groups as specified when specify_groups is multiple specs" do
+      expect(call(3, specify_groups: '1,2|4')).to eq([["1", "2"], ["4"], ["3", "5"]])
+    end
+
+    it "specify_groups aborts when number of specs separated by pipe is out of bounds" do
+      expect do
+        call(3, specify_groups: '1|2|3|4')
+      end.to raise_error(
+        "Number of processes separated by pipe must be less than or equal to the total number of processes"
+      )
+    end
+
+    it "specify_groups aborts when spec passed in doesnt match existing specs" do
+      expect do
+        call(3, specify_groups: '1|2|6')
+      end.to raise_error(
+        "Could not find [\"6\"] from --specify-groups in the main selected files & folders"
+      )
+    end
+
+    it "specify_groups aborts when spec passed in doesnt match existing specs again" do
+      expect do
+        call(3, specify_groups: '1,6|2')
+      end.to raise_error(
+        "Could not find [\"6\"] from --specify-groups in the main selected files & folders"
+      )
+    end
+
+    it "specify_groups aborts when number of specs is equal to number passed in" do
+      expect do
+        call(3, specify_groups: '1|2|3')
+      end.to raise_error(
+        "The number of groups in --specify-groups matches the number of groups from -n but there were other specs found in the main selected files & folders not specified in --specify-groups. Make sure -n is larger than the number of processes in --specify-groups if there are other specs that need to be run. The specs that aren't run: [\"4\", \"5\"]"
+      )
+    end
+
+    it "specify_groups does not abort when the every single spec is specified in it" do
+      expect(call(3, specify_groups: '1,2|3,4|5')).to eq([["1", "2"], ["3", "4"], ["5"]])
+    end
   end
 
   describe '.by_scenarios' do

spec/parallel_tests/test/runner_spec.rb

@@ -178,6 +178,28 @@ def call(*args)
 
         expect(valid_combinations).to include(actual)
       end
+
+      it 'groups by size and uses specified groups of specs in specific order in specific processes' do
+        skip if RUBY_PLATFORM == "java"
+        expect(ParallelTests::Test::Runner).to receive(:runtimes)
+          .and_return({ "aaa1" => 1, "aaa2" => 1, "aaa3" => 2, "bbb" => 3, "ccc" => 1, "ddd" => 2, "eee" => 1 })
+        result = call(
+          ["aaa1", "aaa2", "aaa3", "bbb", "ccc", "ddd", "eee"], 4, specify_groups: 'aaa2,aaa1|bbb', group_by: :runtime
+        )
+
+        specify_groups_1, specify_groups_2, *groups = result
+        expect(specify_groups_1).to eq(["aaa2", "aaa1"])
+        expect(specify_groups_2).to eq(["bbb"])
+        actual = groups.map(&:to_set).to_set
+
+        # both eee and ccs are the same size, so either can be in either group
+        valid_combinations = [
+          [["aaa3", "ccc"], ["ddd", "eee"]].map(&:to_set).to_set,
+          [["aaa3", "eee"], ["ddd", "ccc"]].map(&:to_set).to_set
+        ]
+
+        expect(valid_combinations).to include(actual)
+      end
     end
   end