From 19b1193a1b56ea6fa885f133b4fe5f47450a8b73 Mon Sep 17 00:00:00 2001 From: Eric Kessler Date: Tue, 30 Aug 2022 01:19:54 -0400 Subject: [PATCH] Fix scenario grouping with Rules (#877) * Fix scenario grouping with Rules Updated the scenario detection logic when grouping by scenarios so that it can handle test that are nested under Rules. * Update CHANGELOG Added the scenario grouping Rule fix to the change log. * Fix Rubocop violations Fixed Rubocop violations introduced in the recent commits. * Add development dependencies to gemspec Moving some gems that are definitely needed to develop and test the gem into their proper section of the gemspec instead of them only existing in the development gemfile. * Test against various dependency versions in CI Updated the CI matrix so that the gem is tested against several different versions of its potential runtime dependencies. * Ignore some IDE files Ignoring Jetbrains IDE files. * Fix CI gemfile isues Fixing the paths to the gemspec and the Rubocop task not even having a gemfile specified. * Fix Rubocop violations Fixed Rubocop violations introduced in the recent commits. * Exclude bad CI combinations Not testing against some incompatible dependency combinations. * Fix typo in CI config Fixed a copy/paste mistake. * Fix another typo in CI config Fixed another copy/paste mistake. * Update minimum supported CukeModeler version Removing CukeModeler 2.x from CI testing because this gem requires at least CukeModeler 3.x. * Handle Cucumber Rules better Added a condition to some Rule logic so that older versions of CukeModeler that don't use Rules will still work. * Fix Rubocop violations Fixed Rubocop violations that are showing up in the newer version of Rubocop that is being used in CI. * Fix a test Fixed a test that was using an unreliable way of determining the root directory of the repository. * Remove Cucumber 3 from CI tests Not testing against Cucumber 3 because it has lots of failures and I don't know if it is even officially supported by this gem anymore. * Fix more Rubocop violations Fixed more Rubocop violations that are showing up in the newer version of Rubocop that is being used in CI. * Make a test conditional Not running a test for versions of dependencies for which it is not applicable. * Re-lock dependencies Locking down Cucumber and CukeModeler to their previous version limits because there are lots of test failures in CI for versions that I don't know if this gem even supports. --- .github/workflows/test.yml | 20 ++++++-- .gitignore | 1 + CHANGELOG.md | 2 + Gemfile | 5 -- Gemfile.lock | 22 ++++---- Rakefile | 2 +- ci_gemfiles/cucumber_4.gemfile | 11 ++++ ci_gemfiles/cuke_modeler_3.gemfile | 11 ++++ ci_gemfiles/cuke_modeler_3_minimum.gemfile | 11 ++++ lib/parallel_tests.rb | 6 +-- lib/parallel_tests/cli.rb | 2 +- lib/parallel_tests/cucumber/scenarios.rb | 4 +- lib/parallel_tests/test/runner.rb | 2 +- parallel_tests.gemspec | 7 +++ spec/fixtures/rails70/bin/bundle | 4 +- .../parallel_tests/cucumber/scenarios_spec.rb | 50 +++++++++++++++++++ .../test/runtime_logger_spec.rb | 8 +-- 17 files changed, 139 insertions(+), 29 deletions(-) create mode 100644 ci_gemfiles/cucumber_4.gemfile create mode 100644 ci_gemfiles/cuke_modeler_3.gemfile create mode 100644 ci_gemfiles/cuke_modeler_3_minimum.gemfile diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 37487669d..78498f649 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -12,11 +12,25 @@ jobs: matrix: ruby: ['2.5', '2.6', '2.7', '3.0', '3.1', head, jruby-head] os: [ubuntu-latest, windows-latest] + gemfile: [cucumber_4, cuke_modeler_3_minimum, cuke_modeler_3] task: [spec] include: - - ruby: '2.5' # lowest supported version - os: ubuntu-latest - task: rubocop + - ruby: '2.5' # lowest supported version + os: ubuntu-latest + gemfile: cuke_modeler_3 # Any one of the gemfiles should be fine + task: rubocop + exclude: + # CukeModeler 3.x does not support Ruby 3 until CM 3.6 + - gemfile: cuke_modeler_3_minimum + ruby: jruby-head + - gemfile: cuke_modeler_3_minimum + ruby: head + - gemfile: cuke_modeler_3_minimum + ruby: '3.1' + - gemfile: cuke_modeler_3_minimum + ruby: '3.0' + env: # $BUNDLE_GEMFILE must be set at the job level, so that it is set for all steps + BUNDLE_GEMFILE: ci_gemfiles/${{ matrix.gemfile }}.gemfile steps: - uses: actions/checkout@master - uses: ruby/setup-ruby@v1 diff --git a/.gitignore b/.gitignore index b63e5089b..0e163352f 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,4 @@ tmp .bundle **/vendor/bundle +/.idea diff --git a/CHANGELOG.md b/CHANGELOG.md index 1216a4a4f..733956057 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,8 @@ ### Fixed +- Grouping by scenarios now works for tests that are nested under Rules. + ## 3.11.0 - 2022-05-27 ### Changed diff --git a/Gemfile b/Gemfile index 9f3d04f44..df643bd9c 100644 --- a/Gemfile +++ b/Gemfile @@ -5,9 +5,4 @@ gemspec gem 'bump' gem 'test-unit' gem 'minitest', '~> 5.5.0' -gem 'rspec', '~> 3.3' -gem 'cucumber', "~> 4.0" -gem 'cuke_modeler', '~> 3.0' gem 'spinach', git: "https://github.com/grosser/spinach.git", branch: "grosser/json" # https://github.com/codegram/spinach/pull/229 -gem 'rake' -gem 'rubocop' diff --git a/Gemfile.lock b/Gemfile.lock index ba6afb2b8..e61c4be69 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -62,14 +62,16 @@ GEM cucumber-gherkin (< 23.0) diff-lcs (1.3) ffi (1.15.5) + ffi (1.15.5-x64-mingw32) gherkin-ruby (0.3.2) i18n (1.10.0) concurrent-ruby (~> 1.0) + json (2.6.2) middleware (0.1.0) minitest (5.5.1) multi_test (0.1.2) parallel (1.22.1) - parser (3.1.1.0) + parser (3.1.2.1) ast (~> 2.4.1) power_assert (2.0.1) protobuf-cucumber (3.10.8) @@ -79,7 +81,7 @@ GEM thread_safe rainbow (3.1.1) rake (13.0.6) - regexp_parser (2.2.1) + regexp_parser (2.5.0) rexml (3.2.5) rspec (3.11.0) rspec-core (~> 3.11.0) @@ -94,16 +96,17 @@ GEM diff-lcs (>= 1.2.0, < 2.0) rspec-support (~> 3.11.0) rspec-support (3.11.0) - rubocop (1.26.1) + rubocop (1.35.1) + json (~> 2.3) parallel (~> 1.10) - parser (>= 3.1.0.0) + parser (>= 3.1.2.1) rainbow (>= 2.2.2, < 4.0) regexp_parser (>= 1.8, < 3.0) - rexml - rubocop-ast (>= 1.16.0, < 2.0) + rexml (>= 3.2.5, < 4.0) + rubocop-ast (>= 1.20.1, < 2.0) ruby-progressbar (~> 1.7) unicode-display_width (>= 1.4.0, < 3.0) - rubocop-ast (1.16.0) + rubocop-ast (1.21.0) parser (>= 3.1.1.0) ruby-progressbar (1.11.0) sys-uname (1.2.2) @@ -114,11 +117,12 @@ GEM thread_safe (0.3.6) tzinfo (2.0.4) concurrent-ruby (~> 1.0) - unicode-display_width (2.1.0) + unicode-display_width (2.2.0) zeitwerk (2.5.4) PLATFORMS ruby + x64-mingw32 DEPENDENCIES bump @@ -133,4 +137,4 @@ DEPENDENCIES test-unit BUNDLED WITH - 2.3.7 + 2.3.6 diff --git a/Rakefile b/Rakefile index d184aff78..1409b9f10 100644 --- a/Rakefile +++ b/Rakefile @@ -19,7 +19,7 @@ end desc "bundle all gemfiles [EXTRA=]" task :bundle_all do - extra = ENV["EXTRA"] || "install" + extra = ENV.fetch("EXTRA", nil) || "install" gemfiles = (["Gemfile"] + Dir["spec/fixtures/rails*/Gemfile"]) raise if gemfiles.size < 3 diff --git a/ci_gemfiles/cucumber_4.gemfile b/ci_gemfiles/cucumber_4.gemfile new file mode 100644 index 000000000..fe582868a --- /dev/null +++ b/ci_gemfiles/cucumber_4.gemfile @@ -0,0 +1,11 @@ +# frozen_string_literal: true +source 'https://rubygems.org' +gemspec path: "../" + +gem 'bump' +gem 'test-unit' +gem 'minitest', '~> 5.5.0' +gem 'spinach', git: "https://github.com/grosser/spinach.git", branch: "grosser/json" # https://github.com/codegram/spinach/pull/229 + +# The version of Cucumber being tested +gem 'cucumber', '~> 4.0' diff --git a/ci_gemfiles/cuke_modeler_3.gemfile b/ci_gemfiles/cuke_modeler_3.gemfile new file mode 100644 index 000000000..6b8175385 --- /dev/null +++ b/ci_gemfiles/cuke_modeler_3.gemfile @@ -0,0 +1,11 @@ +# frozen_string_literal: true +source 'https://rubygems.org' +gemspec path: "../" + +gem 'bump' +gem 'test-unit' +gem 'minitest', '~> 5.5.0' +gem 'spinach', git: "https://github.com/grosser/spinach.git", branch: "grosser/json" # https://github.com/codegram/spinach/pull/229 + +# The version of CukeModeler being tested +gem 'cuke_modeler', '~> 3.0' diff --git a/ci_gemfiles/cuke_modeler_3_minimum.gemfile b/ci_gemfiles/cuke_modeler_3_minimum.gemfile new file mode 100644 index 000000000..294409767 --- /dev/null +++ b/ci_gemfiles/cuke_modeler_3_minimum.gemfile @@ -0,0 +1,11 @@ +# frozen_string_literal: true +source 'https://rubygems.org' +gemspec path: "../" + +gem 'bump' +gem 'test-unit' +gem 'minitest', '~> 5.5.0' +gem 'spinach', git: "https://github.com/grosser/spinach.git", branch: "grosser/json" # https://github.com/codegram/spinach/pull/229 + +# The version of CukeModeler being tested +gem 'cuke_modeler', '3.0.0' diff --git a/lib/parallel_tests.rb b/lib/parallel_tests.rb index 356ba7eff..b96162be3 100644 --- a/lib/parallel_tests.rb +++ b/lib/parallel_tests.rb @@ -16,7 +16,7 @@ class << self def determine_number_of_processes(count) [ count, - ENV["PARALLEL_TEST_PROCESSORS"], + ENV.fetch("PARALLEL_TEST_PROCESSORS", nil), Parallel.processor_count ].detect { |c| !c.to_s.strip.empty? }.to_i end @@ -68,8 +68,8 @@ def first_process? end def last_process? - current_process_number = ENV['TEST_ENV_NUMBER'] - total_processes = ENV['PARALLEL_TEST_GROUPS'] + current_process_number = ENV.fetch('TEST_ENV_NUMBER', nil) + total_processes = ENV.fetch('PARALLEL_TEST_GROUPS', nil) return true if current_process_number.nil? && total_processes.nil? current_process_number = '1' if current_process_number.nil? current_process_number == total_processes diff --git a/lib/parallel_tests/cli.rb b/lib/parallel_tests/cli.rb index 6c0e6928b..96daa3236 100644 --- a/lib/parallel_tests/cli.rb +++ b/lib/parallel_tests/cli.rb @@ -383,7 +383,7 @@ def use_colors? end def first_is_1? - val = ENV["PARALLEL_TEST_FIRST_IS_1"] + val = ENV.fetch("PARALLEL_TEST_FIRST_IS_1", nil) ['1', 'true'].include?(val) end diff --git a/lib/parallel_tests/cucumber/scenarios.rb b/lib/parallel_tests/cucumber/scenarios.rb index 15852cdcd..6c95998fe 100644 --- a/lib/parallel_tests/cucumber/scenarios.rb +++ b/lib/parallel_tests/cucumber/scenarios.rb @@ -52,7 +52,9 @@ def split_into_scenarios(files, tags = '') feature_tags = feature.tags.map(&:name) # We loop on each children of the feature - feature.tests.each do |test| + test_models = feature.tests + test_models += feature.rules.map(&:tests).flatten if feature.respond_to?(:rules) + test_models.each do |test| # It's a scenario, we add it to the scenario_line_logger scenario_line_logger.visit_feature_element(document.path, test, feature_tags, line_numbers: test_lines) end diff --git a/lib/parallel_tests/test/runner.rb b/lib/parallel_tests/test/runner.rb index 836630af1..46e8b62da 100644 --- a/lib/parallel_tests/test/runner.rb +++ b/lib/parallel_tests/test/runner.rb @@ -154,7 +154,7 @@ def command_with_seed(cmd, seed) def executable if ENV.include?('PARALLEL_TESTS_EXECUTABLE') - [ENV['PARALLEL_TESTS_EXECUTABLE']] + [ENV.fetch('PARALLEL_TESTS_EXECUTABLE')] else determine_executable end diff --git a/parallel_tests.gemspec b/parallel_tests.gemspec index b6d5f3f9d..45093e0b6 100644 --- a/parallel_tests.gemspec +++ b/parallel_tests.gemspec @@ -17,5 +17,12 @@ Gem::Specification.new name, ParallelTests::VERSION do |s| s.license = "MIT" s.executables = ["parallel_spinach", "parallel_cucumber", "parallel_rspec", "parallel_test"] s.add_runtime_dependency "parallel" + + s.add_development_dependency 'cucumber', '~> 4.0' + s.add_development_dependency 'cuke_modeler', '~> 3.0' + s.add_development_dependency "rake" + s.add_development_dependency 'rspec', '~> 3.3' + s.add_development_dependency "rubocop" + s.required_ruby_version = '>= 2.5.0' end diff --git a/spec/fixtures/rails70/bin/bundle b/spec/fixtures/rails70/bin/bundle index 25dfd64d8..650643e31 100755 --- a/spec/fixtures/rails70/bin/bundle +++ b/spec/fixtures/rails70/bin/bundle @@ -18,7 +18,7 @@ m = Module.new do end def env_var_version - ENV["BUNDLER_VERSION"] + ENV.fetch("BUNDLER_VERSION", nil) end def cli_arg_version @@ -38,7 +38,7 @@ m = Module.new do end def gemfile - gemfile = ENV["BUNDLE_GEMFILE"] + gemfile = ENV.fetch("BUNDLE_GEMFILE", nil) return gemfile if gemfile && !gemfile.empty? File.expand_path('../Gemfile', __dir__) diff --git a/spec/parallel_tests/cucumber/scenarios_spec.rb b/spec/parallel_tests/cucumber/scenarios_spec.rb index 4799b31c3..88bd42a4a 100644 --- a/spec/parallel_tests/cucumber/scenarios_spec.rb +++ b/spec/parallel_tests/cucumber/scenarios_spec.rb @@ -232,4 +232,54 @@ expect(scenarios.length).to eq 1 end end + + cuke_modeler_version = Gem.loaded_specs['cuke_modeler'].version.version + major = cuke_modeler_version.match(/^(\d+)\./)[1].to_i + minor = cuke_modeler_version.match(/^\d+\.(\d+)\./)[1].to_i + + # CukeModeler doesn't support Rule models until 3.2.0 + context 'with Rules', if: (major > 3) || (minor >= 2) do + let(:feature_file) do + Tempfile.new('grouper.feature').tap do |feature| + feature.write <<-EOS + Feature: Grouping by scenario + + Scenario: First + Given I do nothing + + Scenario Outline: Second + Given I don't do anything + Examples: + | param | + | value 1 | + | value 2 | + + Rule: + Scenario: Third + Given I don't do anything + + Rule: + Scenario Outline: Fourth + Given I don't do anything + Examples: + | param | + | value 1 | + | value 2 | + EOS + feature.rewind + end + end + + it 'returns all the scenarios' do + scenarios = ParallelTests::Cucumber::Scenarios.all([feature_file.path]) + expect(scenarios).to match_array [ + "#{feature_file.path}:3", + "#{feature_file.path}:10", + "#{feature_file.path}:11", + "#{feature_file.path}:14", + "#{feature_file.path}:22", + "#{feature_file.path}:23" + ] + end + end end diff --git a/spec/parallel_tests/test/runtime_logger_spec.rb b/spec/parallel_tests/test/runtime_logger_spec.rb index f609fdd53..0e038c0c6 100644 --- a/spec/parallel_tests/test/runtime_logger_spec.rb +++ b/spec/parallel_tests/test/runtime_logger_spec.rb @@ -7,12 +7,14 @@ def run(command) raise "FAILED: #{result}" unless $?.success? end - def run_tests - run ["ruby", "#{Bundler.root}/bin/parallel_test", "test", "-n", "2"] + def run_tests(repo_root_dir) + run ["ruby", "#{repo_root_dir}/bin/parallel_test", "test", "-n", "2"] end it "writes a correct log on minitest-5" do skip if RUBY_PLATFORM == "java" # just too slow ... + repo_root = Dir.pwd + use_temporary_directory do # setup simple structure FileUtils.mkdir "test" @@ -37,7 +39,7 @@ def test_foo RUBY end - run_tests + run_tests(repo_root) # log looking good ? lines = File.read("tmp/parallel_runtime_test.log").split("\n").sort.map { |x| x.sub(/\d$/, "") }