[Fix rubocop#117] Add --parallel option

This change tries to solve the tricky parallel execution problem
by spawning off a number of processes/threads to do file inspection,
sharing the work between them, without collecting any output.
When all processes are finished, the original process runs the
full inspection again, taking advantage of result caching.

CHANGELOG.md

@@ -2,6 +2,10 @@
 
 ## master (unreleased)
 
+### New features
+
+* [#117](https://github.com/bbatsov/rubocop/issues/117): Add `--parallel` option for running RuboCop in multiple processes or threads. ([@jonas054][])
+
 ### Changes
 
 * [#4262](https://github.com/bbatsov/rubocop/pull/4262): Add new `MinSize` configuration to `Style/SymbolArray`, consistent with the same configuration in `Style/WordArray`. ([@scottmatthewman][])

lib/rubocop/cli.rb

@@ -21,6 +21,7 @@ def initialize
     # @return [Integer] UNIX exit code
     def run(args = ARGV)
       @options, paths = Options.new.parse(args)
+      validate_options_vs_config
       act_on_options
       apply_default_formatter
 
@@ -47,6 +48,15 @@ def trap_interrupt(runner)
 
     private
 
+    def validate_options_vs_config
+      if @options[:parallel] &&
+         !@config_store.for(Dir.pwd).for_all_cops['UseCache']
+        raise ArgumentError, '-P/--parallel uses caching to speed up ' \
+                             'execution, so combining with AllCops: ' \
+                             'UseCache: false is not allowed.'
+      end
+    end
+
     def act_on_options
       handle_exiting_options
 

lib/rubocop/options.rb

@@ -156,6 +156,7 @@ def add_boolean_flags(opts) # rubocop:disable Metrics/MethodLength
 
       option(opts, '-v', '--version')
       option(opts, '-V', '--verbose-version')
+      option(opts, '-P', '--parallel')
     end
 
     def add_list_options(opts)
@@ -217,11 +218,25 @@ def validate_compatibility # rubocop:disable Metrics/MethodLength
         raise ArgumentError, '--no-offense-counts can only be used together ' \
                              'with --auto-gen-config.'
       end
+      validate_parallel
+
       return if incompatible_options.size <= 1
       raise ArgumentError, 'Incompatible cli options: ' \
                            "#{incompatible_options.inspect}"
     end
 
+    def validate_parallel
+      if parallel_without_caching?
+        raise ArgumentError, '-P/--parallel uses caching to speed up ' \
+                             'execution, so combining with --cache false is ' \
+                             'not allowed.'
+      end
+
+      return unless parallel_with_autocorrect?
+      raise ArgumentError, '-P/--parallel can not be combined with ' \
+                           '--auto-correct.'
+    end
+
     def only_includes_unneeded_disable?
       @options.key?(:only) &&
         (@options[:only] & %w[Lint/UnneededDisable UnneededDisable]).any?
@@ -240,6 +255,14 @@ def no_offense_counts_without_auto_gen_config?
       @options.key?(:no_offense_counts) && !@options.key?(:auto_gen_config)
     end
 
+    def parallel_without_caching?
+      @options.key?(:parallel) && @options[:cache] == 'false'
+    end
+
+    def parallel_with_autocorrect?
+      @options.key?(:parallel) && @options.key?(:auto_correct)
+    end
+
     def incompatible_options
       @incompatible_options ||= @options.keys & Options::EXITING_OPTIONS
     end
@@ -323,6 +346,8 @@ module OptionsHelp
       no_color:              'Force color output on or off.',
       version:               'Display version.',
       verbose_version:       'Display verbose version.',
+      parallel:             ['Use available CPUs to execute inspection in',
+                             'parallel.'],
       stdin:                ['Pipe source from STDIN, using FILE in offense',
                              'reports. This is useful for editor integration.']
     }.freeze

lib/rubocop/result_cache.rb

@@ -9,7 +9,7 @@ module RuboCop
   # Provides functionality for caching rubocop runs.
   class ResultCache
     NON_CHANGING = %i[color format formatters out debug fail_level
-                      cache fail_fast stdin].freeze
+                      cache fail_fast stdin parallel].freeze
 
     # Remove old files so that the cache doesn't grow too big. When the
     # threshold MaxFilesInCache has been exceeded, the oldest 50% of all the

lib/rubocop/runner.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require 'parallel'
+
 module RuboCop
   # This class handles the processing of files, which includes dealing with
   # formatters and letting cops inspect the files.
@@ -33,6 +35,7 @@ def run(paths)
       if @options[:list_target_files]
         list_files(target_files)
       else
+        warm_cache(target_files) if @options[:parallel]
         inspect_files(target_files)
       end
     end
@@ -43,6 +46,13 @@ def abort
 
     private
 
+    # Warms up the RuboCop cache by forking a suitable number of rubocop
+    # instances that each inspects its alotted group of files.
+    def warm_cache(target_files)
+      puts 'Running parallel inspection'
+      Parallel.each(target_files, &method(:file_offenses))
+    end
+
     def find_target_files(paths)
       target_finder = TargetFinder.new(@config_store, @options)
       target_files = target_finder.find(paths)

manual/basic_usage.md

@@ -79,6 +79,7 @@ Command flag              | Description
 `--fail-level`            | Minimum [severity](#severity) for exit with error code. Full severity name or upper case initial can be given. Normally, auto-corrected offenses are ignored. Use `A` or `autocorrect` if you'd like them to trigger failure.
 `-s/--stdin`              | Pipe source from STDIN. This is useful for editor integration.
 `--[no-]color`            | Force color output on or off.
+`--parallel`              | Use available CPUs to execute inspection in parallel.
 
 Default command-line options are loaded from `.rubocop` and `RUBOCOP_OPTS` and are combined with command-line options that are explicitly passed to `rubocop`.
 Thus, the options have the following order of precedence (from highest to lowest):

rubocop.gemspec

@@ -30,6 +30,7 @@ Gem::Specification.new do |s|
   s.add_runtime_dependency('powerpack', '~> 0.1')
   s.add_runtime_dependency('ruby-progressbar', '~> 1.7')
   s.add_runtime_dependency('unicode-display_width', '~> 1.0', '>= 1.0.1')
+  s.add_runtime_dependency('parallel', '~> 1.10')
 
   s.add_development_dependency('bundler', '~> 1.3')
 end

spec/rubocop/cli/cli_options_spec.rb

@@ -9,6 +9,39 @@
     RuboCop::ConfigLoader.default_configuration = nil
   end
 
+  describe '--parallel' do
+    if RuboCop::Platform.windows?
+      context 'on Windows' do
+        it 'prints a warning' do
+          cli.run ['-P']
+          expect($stderr.string)
+            .to include('Process.fork is not supported by this Ruby')
+        end
+      end
+    else
+      context 'combined with AllCops:UseCache:false' do
+        before do
+          create_file('.rubocop.yml', ['AllCops:',
+                                       '  UseCache: false'])
+        end
+        it 'fails with an error message' do
+          cli.run %w[-P]
+          expect($stderr.string)
+            .to include('-P/--parallel uses caching to speed up execution, ' \
+                        'so combining with AllCops: UseCache: false is not ' \
+                        'allowed.')
+        end
+      end
+
+      context 'on Unix-like systems' do
+        it 'prints a message' do
+          cli.run ['--parallel']
+          expect($stdout.string).to match(/Running parallel inspection/)
+        end
+      end
+    end
+  end
+
   describe '--list-target-files' do
     context 'when there are no files' do
       it 'prints nothing with -L' do

spec/rubocop/options_spec.rb

@@ -93,6 +93,8 @@ def abs(path)
                   --[no-]color                 Force color output on or off.
               -v, --version                    Display version.
               -V, --verbose-version            Display verbose version.
+              -P, --parallel                   Use available CPUs to execute inspection in
+                                               parallel.
               -s, --stdin FILE                 Pipe source from STDIN, using FILE in offense
                                                reports. This is useful for editor integration.
         END
@@ -153,6 +155,25 @@ def abs(path)
       end
     end
 
+    describe '--parallel' do
+      context 'combined with --cache false' do
+        it 'fails with an error message' do
+          msg = ['-P/--parallel uses caching to speed up execution, so ',
+                 'combining with --cache false is not allowed.'].join
+          expect { options.parse %w[--parallel --cache false] }
+            .to raise_error(ArgumentError, msg)
+        end
+      end
+
+      context 'combined with --auto-correct' do
+        it 'fails with an error message' do
+          msg = '-P/--parallel can not be combined with --auto-correct.'
+          expect { options.parse %w[--parallel --auto-correct] }
+            .to raise_error(ArgumentError, msg)
+        end
+      end
+    end
+
     describe '--fail-level' do
       it 'accepts full severity names' do
         %w[refactor convention warning error fatal].each do |severity|