diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2e7b9ccb..801ee02a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -2,13 +2,31 @@ name: CI on: [push] +env: + RAILS_ENV: test + CC_TEST_REPORTER_ID: ${{ secrets.CC_TEST_REPORTER_ID }} + jobs: - build: + linters: + name: Linters + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v3 + - name: Setup Ruby + uses: ruby/setup-ruby@v1 + with: + bundler-cache: true + - name: Run Code Analysis + run: | + bundle exec rake code:analysis + tests: + name: Tests runs-on: ubuntu-latest services: db: - image: postgres:9.4 + image: postgres:15 env: POSTGRES_USER: postgres POSTGRES_HOST_AUTH_METHOD: trust @@ -19,10 +37,17 @@ jobs: --health-interval 10s --health-timeout 5s --health-retries 5 - - env: - RAILS_ENV: test - CC_TEST_REPORTER_ID: ${{ secrets.CC_TEST_REPORTER_ID }} + + strategy: + fail-fast: false + matrix: + # Set N number of parallel jobs you want to run tests on. + # Use higher number if you have slow tests to split them on more parallel jobs. + # Remember to update ci_node_index below to 0..N-1 + ci_node_total: [1] + # set N-1 indexes for parallel jobs + # When you run 2 parallel jobs then first job will have index 0, the second job will have index 1 etc + ci_node_index: [0] steps: - name: Checkout code @@ -35,21 +60,25 @@ jobs: run: | curl -L https://codeclimate.com/downloads/test-reporter/test-reporter-latest-linux-amd64 > ./cc-test-reporter chmod +x ./cc-test-reporter - ./cc-test-reporter before-build + ./cc-test-reporter before-build - name: Setup Database - run: bundle exec rails db:create db:migrate + run: | + bundle exec rake parallel:load_schema - name: Check for untracked changes in schema.rb uses: rootstrap/check_untracked_changes@v1 with: path: "./db/schema.rb" - - name: I18n Health - run: bundle exec i18n-tasks health - - name: Run Code Analysis + - name: Get CPU info + id: cpu_info run: | - bundle exec rake code:analysis + echo "cpu_cores=$(nproc)" >> $GITHUB_ENV - name: Run Tests + env: + KNAPSACK_CI_NODE_TOTAL: ${{ matrix.ci_node_total }} + KNAPSACK_CI_NODE_INDEX: ${{ matrix.ci_node_index }} + PARALLEL_TESTS_CONCURRENCY: ${{ env.cpu_cores }} run: | - bundle exec rspec + bundle exec parallel_rspec -n $PARALLEL_TESTS_CONCURRENCY -e './bin/parallel_tests' - name: Check for missing annotations run: bundle exec annotate - name: Check for untracked changes in app and spec directories @@ -58,4 +87,27 @@ jobs: path: "./app/ ./spec/" - name: Report to CodeClimate run: | - ./cc-test-reporter after-build --exit-code 0 + ./cc-test-reporter format-coverage --output "coverage/coverage.${{ matrix.ci_node_index }}.json" + - name: Upload partial converage + uses: actions/upload-artifact@v2 + with: + name: coverage + path: "coverage/coverage.${{ matrix.ci_node_index }}.json" + coverage: + name: Coverage + runs-on: ubuntu-latest + needs: tests + steps: + - name: Setup Code Climate test-reporter + run: | + curl -L https://codeclimate.com/downloads/test-reporter/test-reporter-latest-linux-amd64 > ./cc-test-reporter + chmod +x ./cc-test-reporter + - name: download coverage reports + uses: actions/download-artifact@v2 + with: + name: coverage + path: coverage/coverage.*.json + - name: report coverage + run: | + ./cc-test-reporter sum-coverage coverage/**/*.json + ./cc-test-reporter upload-coverage diff --git a/.github/workflows/update_knapsack_report.yml b/.github/workflows/update_knapsack_report.yml new file mode 100644 index 00000000..c1d54b4f --- /dev/null +++ b/.github/workflows/update_knapsack_report.yml @@ -0,0 +1,63 @@ +name: Update Knapsack Report + +# This workflow should be scheduled at certain intervals +on: + schedule: + - cron: '0 5 31 2 *' +# The above cron does not run. Replace with the wanted periodicity. +# For example, the following configuration would schedule it every 3 months +# - cron: '0 0 1 */3 *' + +jobs: + build: + runs-on: ubuntu-latest + + services: + db: + image: postgres:15 + env: + POSTGRES_USER: postgres + POSTGRES_HOST_AUTH_METHOD: trust + ports: + - 5432:5432 + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 + + env: + RAILS_ENV: test + + steps: + - uses: actions/checkout@v2 + - name: Set up Ruby 3.1 + uses: actions/setup-ruby@v1 + with: + ruby-version: 3.1 + github_token: ${{ secrets.GITHUB_TOKEN }} + - name: Setup Database + run: bundle exec rails db:create db:migrate + - name: Update Knapsack Report + run: KNAPSACK_GENERATE_REPORT=true bundle exec rspec + - name: Commit files + run: | + git config --local user.email "action@github.com" + git config --local user.name "GitHub Action" + git add knapsack_rspec_report.json + git commit -m "Update knapsack_rspec_report.json" + - name: Push changes + run: git push origin HEAD:update-knapsack-report + - name: Initialize Pull Request + uses: rootstrap/create-pull-request@v3 + with: + pull_request_token: ${{ secrets.GITHUB_TOKEN }} + head: update-knapsack-report + base: main + title: 'Update Knapsack report' + maintainer_can_modify: true + draft: false + body: > + This is an automated PR. Knapsack uses the report file to distribute tests across different nodes + so that they all have similar execution times. It's advisable to update the report periodically, + especially if many test files have been added, for better distribution. diff --git a/Gemfile b/Gemfile index 91d472ce..37397ca1 100644 --- a/Gemfile +++ b/Gemfile @@ -41,6 +41,8 @@ group :development, :test do gem 'annotate', '~> 3.2', '>= 3.0.3' gem 'dotenv-rails', '~> 2.7.6' gem 'factory_bot_rails', '~> 6.2' + gem 'knapsack', '~> 4.0' + gem 'parallel_tests', '~> 4.2' gem 'pry-byebug', '~> 3.9', platform: :mri gem 'pry-rails', '~> 0.3.9' gem 'rspec_api_documentation', '~> 6.1.0' diff --git a/Gemfile.lock b/Gemfile.lock index 83dff301..6efbfefd 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -118,6 +118,7 @@ GEM aws-sigv4 (~> 1.6) aws-sigv4 (1.6.0) aws-eventstream (~> 1, >= 1.0.2) + base64 (0.1.1) bcrypt (3.1.18) better_errors (2.10.1) erubi (>= 1.0.0) @@ -248,6 +249,8 @@ GEM activerecord kaminari-core (= 1.2.2) kaminari-core (1.2.2) + knapsack (4.0.0) + rake kwalify (0.7.2) language_server-protocol (3.17.0.3) launchy (2.5.0) @@ -293,7 +296,9 @@ GEM oj (3.16.1) orm_adapter (0.5.0) pagy (4.11.0) - parallel (1.23.0) + parallel (1.22.1) + parallel_tests (4.2.1) + parallel parser (3.2.2.3) ast (~> 2.4.1) racc @@ -408,7 +413,8 @@ GEM activesupport (>= 3.0.0) mustache (~> 1.0, >= 0.99.4) rspec (~> 3.0) - rubocop (1.55.0) + rubocop (1.56.0) + base64 (~> 0.1.1) json (~> 2.3) language_server-protocol (>= 3.17.0) parallel (~> 1.10) @@ -524,11 +530,13 @@ DEPENDENCIES flipper-ui (~> 0.28.0) i18n-tasks (~> 1.0.12) jbuilder (~> 2.10) + knapsack (~> 4.0) letter_opener (~> 1.7) listen (~> 3.8) lograge (~> 0.13) oj (~> 3.16) pagy (~> 4.0) + parallel_tests (~> 4.2) pg (~> 1.5) pg_query (~> 4.2.1) prosopite (~> 1.3.2) diff --git a/README.md b/README.md index e106c562..2eac4182 100644 --- a/README.md +++ b/README.md @@ -81,9 +81,11 @@ To illustrate, `bin/rails console` will run the console in the docker container - [Faker](https://github.com/stympy/faker) for generating test data - [Flipper](https://github.com/jnunemaker/flipper) for feature flag support - [Jbuilder](https://github.com/rails/jbuilder) for json views +- [Knapsack](https://github.com/KnapsackPro/knapsack) for generating CI time report and splitting into nodes - [Letter Opener](https://github.com/ryanb/letter_opener) for previewing a mail in the browser - [Oj](https://github.com/ohler55/oj) for optimized json - [Pagy](https://github.com/ddnexus/pagy) for pagination +- [Parallel Tests](https://github.com/grosser/parallel_tests) for split CI in several cores - [Prosopite](https://github.com/charkost/prosopite) to detect N+1 queries - [Pry](https://github.com/pry/pry) for enhancing the ruby shell - [Puma](https://github.com/puma/puma) for the server @@ -105,6 +107,7 @@ To illustrate, `bin/rails console` will run the console in the docker container - Set your [frontend URL](https://github.com/cyu/rack-cors#origin) in `config/initializers/rack_cors.rb` - Set your mail sender in `config/initializers/devise.rb` - Config your timezone accordingly in `application.rb`. +- Config CI in parallel execution. See [docs](docs/ci.md). ## Api Docs diff --git a/Rakefile b/Rakefile index 488c551f..34b92f93 100644 --- a/Rakefile +++ b/Rakefile @@ -6,3 +6,5 @@ require_relative 'config/application' Rails.application.load_tasks + +Knapsack.load_tasks if defined?(Knapsack) diff --git a/bin/parallel_tests b/bin/parallel_tests new file mode 100755 index 00000000..fdbd945f --- /dev/null +++ b/bin/parallel_tests @@ -0,0 +1,18 @@ +#!/bin/bash +# This file should be in bin/parallel_tests + +# updates CI node total based on parallel_tests concurrency +KNAPSACK_CI_NODE_TOTAL=$(( $PARALLEL_TESTS_CONCURRENCY * $KNAPSACK_CI_NODE_TOTAL )) + +if [ "$TEST_ENV_NUMBER" == "" ]; then + PARALLEL_TESTS_CONCURRENCY_INDEX=0 +else + PARALLEL_TESTS_CONCURRENCY_INDEX=$(( $TEST_ENV_NUMBER - 1 )) +fi + +KNAPSACK_CI_NODE_INDEX=$(( $PARALLEL_TESTS_CONCURRENCY_INDEX + ($PARALLEL_TESTS_CONCURRENCY * $KNAPSACK_CI_NODE_INDEX) )) + +# logs info about ENVs to ensure everything works +echo KNAPSACK_CI_NODE_TOTAL=$KNAPSACK_CI_NODE_TOTAL KNAPSACK_CI_NODE_INDEX=$KNAPSACK_CI_NODE_INDEX PARALLEL_TESTS_CONCURRENCY=$PARALLEL_TESTS_CONCURRENCY + +CI_NODE_TOTAL=$KNAPSACK_CI_NODE_TOTAL CI_NODE_INDEX=$KNAPSACK_CI_NODE_INDEX bundle exec rake knapsack:rspec diff --git a/config/database.yml b/config/database.yml index 80903b9b..9020ee02 100644 --- a/config/database.yml +++ b/config/database.yml @@ -13,7 +13,7 @@ development: test: <<: *default - database: rails_api_base_test + database: rails_api_base_test<%= ENV['TEST_ENV_NUMBER'] %> # For production is recommended to set DATABASE_URL env variable. # It will take precedence over the config defined here. diff --git a/docs/ci.md b/docs/ci.md new file mode 100644 index 00000000..d2960f44 --- /dev/null +++ b/docs/ci.md @@ -0,0 +1,44 @@ +# CI + +## Parallelization with Parallel Tests & Knapsack +Knapsack and Parallel Tests gems allow us to run tests in several nodes at the same time, benefiting us in the execution time. Knapsack parallelizes them at node level while Parallel Tests does it at CPU level. + +Knapsack splits tests based on an execution time report. In case there are files that were not added in the report, they will all run on the same node and may overload it, so it is strongly recommended to update the report frequently. + +## Configuration +In case you want to use this you will need the script that splits spec files called `parallel_tests`, which sets up the configuration, assuming you have `n_nodes * cpu_cores_quantity`. + +On Github Actions you can add any nodes you want using matrix strategy, setting up some variables: + +```sh + ci_node_total: [4] + # set N-1 indexes for parallel jobs + # When you run 2 parallel jobs then first job will have index 0, the second job will have index 1 etc + ci_node_index: [0, 1, 2, 3] +``` + +CPU cores quantity on every node are obtained automatically from Github Actions config `echo "cpu_cores=$(nproc)" >> $GITHUB_ENV`. + +If you want to update it manually you can do it by updating this variable: +`PARALLEL_TESTS_CONCURRENCY: 2` + +To update tests on local machine you can execute `KNAPSACK_CI_NODE_TOTAL=4 KNAPSACK_CI_NODE_INDEX=1 PARALLEL_TESTS_CONCURRENCY=2 bundle exec parallel_rspec -n 2 -e './bin/parallel_tests'`. This will run subset of tests files corresponding to second node. + +## Generating report +Knapsack report needs to be updated frequently to balance execution time among nodes. This can be done manually by executing: +`KNAPSACK_GENERATE_REPORT=true bundle exec rspec` + +It is also recommended to generate the report in the CI for a better precision. For this you have available a workflow in Github Actions that triggers the report generation and creates a pull request automatically. This workflow can be scheduled in the frequency you want or even can be manually triggered. + +To schedule the cron task you have to do it in `.github/workflows/update_knapsack_report.yml:6` +It is now scheduled for February 31 so will never run. + +```sh + - cron: '0 5 31 2 *' + # The above cron does not run. Replace with the wanted periodicity. +``` +## Coverage +When splitting tests in different nodes, each report covers only a part of the code files being tested. +For this reason a job in the CI is added to sums coverages from all nodes to be used by SimpleCov. This job will be executed after all nodes have finished and will send the final report to CodeClimate. + +For the case of CPU cores we do not need to add extra configuration since the report of each node contains the info of all the cores that have been splited. diff --git a/knapsack_rspec_report.json b/knapsack_rspec_report.json new file mode 100644 index 00000000..cde01527 --- /dev/null +++ b/knapsack_rspec_report.json @@ -0,0 +1,31 @@ +{ + "spec/policies/admin/page_policy_spec.rb": 0.038240999972913414, + "spec/routing/sessions_routing_spec.rb": 0.006703999999444932, + "spec/requests/api/v1/users/create_spec.rb": 0.5093869999982417, + "spec/acceptance/settings_spec.rb": 0.17437299998709932, + "spec/acceptance/sessions_spec.rb": 0.5090029999846593, + "spec/acceptance/status_spec.rb": 0.009465000010095537, + "spec/requests/api/v1/sessions/destroy_spec.rb": 0.2923869999940507, + "spec/policies/admin/user_policy_spec.rb": 0.020087999990209937, + "spec/routing/registrations_routing_spec.rb": 0.001361000002361834, + "spec/policies/admin/application_policy_spec.rb": 0.011096000031102449, + "spec/policies/admin_user_policy_spec.rb": 0.009194999991450459, + "spec/routing/user_routing_spec.rb": 0.0026309999520890415, + "spec/requests/api/v1/passwords/update_spec.rb": 0.4883949999930337, + "spec/requests/api/v1/feature_flags_spec.rb": 0.040616000012960285, + "spec/requests/api/v1/settings_spec.rb": 0.0425929999910295, + "spec/requests/api/v1/passwords/create_spec.rb": 0.15221400000154972, + "spec/acceptance/users_spec.rb": 0.5729850000352599, + "spec/policies/user_policy_spec.rb": 0.015309000038541853, + "spec/requests/api/v1/users/update_spec.rb": 0.9842829999979585, + "spec/requests/api/v1/status_spec.rb": 0.01666500000283122, + "spec/policies/application_policy_spec.rb": 0.008917000028304756, + "spec/policies/admin/admin_user_policy_spec.rb": 0.005394000036176294, + "spec/requests/api/v1/sessions/create_spec.rb": 0.49404099996900186, + "spec/decorators/user_decorator_spec.rb": 0.005971000005956739, + "spec/requests/api/v1/passwords/edit_spec.rb": 0.08089700003620237, + "spec/acceptance/passwords_spec.rb": 0.6197870000032708, + "spec/models/user_spec.rb": 0.09392800001660362, + "spec/mailers/application_mailer_spec.rb": 0.0020680000307038426, + "spec/requests/api/v1/users/show_spec.rb": 0.5047900000354275 +} diff --git a/lib/tasks/code_analysis.rake b/lib/tasks/code_analysis.rake index e6dc69d8..d9e5020a 100644 --- a/lib/tasks/code_analysis.rake +++ b/lib/tasks/code_analysis.rake @@ -7,5 +7,6 @@ namespace :code do sh 'bundle exec rubocop .' sh 'bundle exec reek .' sh 'bundle exec rails_best_practices .' + sh 'bundle exec i18n-tasks health' end end diff --git a/spec/rails_helper.rb b/spec/rails_helper.rb index 204da860..5c8db7c2 100644 --- a/spec/rails_helper.rb +++ b/spec/rails_helper.rb @@ -11,6 +11,7 @@ add_filter 'app/admin' add_filter 'config' add_filter 'spec' + add_filter 'lib/tasks/code_analysis.rake' end require File.expand_path('../config/environment', __dir__) diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb index cde9663e..0c5d6c63 100644 --- a/spec/spec_helper.rb +++ b/spec/spec_helper.rb @@ -6,10 +6,14 @@ require 'factory_bot_rails' require 'helpers' +require 'knapsack' require 'webmock/rspec' require 'shoulda/matchers' require 'pundit/rspec' +Knapsack.tracker.config(enable_time_offset_warning: false) +Knapsack::Adapters::RSpecAdapter.bind + FactoryBot.factories.clear FactoryBot.reload