Release/improve ci

.github/workflows/ci.yml

@@ -2,13 +2,31 @@ name: CI
 
 on: [push]
 
+env:
+  RAILS_ENV: test
+  CC_TEST_REPORTER_ID: ${{ secrets.CC_TEST_REPORTER_ID }}
+
 jobs:
-  build:
+  linters:
+    name: Linters
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v3
+      - name: Setup Ruby
+        uses: ruby/setup-ruby@v1
+        with:
+          bundler-cache: true
+      - name: Run Code Analysis
+        run: |
+          bundle exec rake code:analysis
+  tests:
+    name: Tests
     runs-on: ubuntu-latest
 
     services:
       db:
-        image: postgres:9.4
+        image: postgres:15
         env:
           POSTGRES_USER: postgres
           POSTGRES_HOST_AUTH_METHOD: trust
@@ -19,10 +37,17 @@ jobs:
           --health-interval 10s
           --health-timeout 5s
           --health-retries 5
-
-    env:
-      RAILS_ENV: test
-      CC_TEST_REPORTER_ID: ${{ secrets.CC_TEST_REPORTER_ID }}
+
+    strategy:
+      fail-fast: false
+      matrix:
+        # Set N number of parallel jobs you want to run tests on.
+        # Use higher number if you have slow tests to split them on more parallel jobs.
+        # Remember to update ci_node_index below to 0..N-1
+        ci_node_total: [1]
+        # set N-1 indexes for parallel jobs
+        # When you run 2 parallel jobs then first job will have index 0, the second job will have index 1 etc
+        ci_node_index: [0]
 
     steps:
       - name: Checkout code
@@ -35,21 +60,25 @@ jobs:
         run: |
           curl -L https://codeclimate.com/downloads/test-reporter/test-reporter-latest-linux-amd64 > ./cc-test-reporter
           chmod +x ./cc-test-reporter
-          ./cc-test-reporter before-build        
+          ./cc-test-reporter before-build
       - name: Setup Database
-        run: bundle exec rails db:create db:migrate
+        run: |
+          bundle exec rake parallel:load_schema
       - name: Check for untracked changes in schema.rb
         uses: rootstrap/check_untracked_changes@v1
         with:
           path: "./db/schema.rb"
-      - name: I18n Health
-        run: bundle exec i18n-tasks health
-      - name: Run Code Analysis
+      - name: Get CPU info
+        id: cpu_info
         run: |
-          bundle exec rake code:analysis
+          echo "cpu_cores=$(nproc)" >> $GITHUB_ENV
       - name: Run Tests
+        env:
+          KNAPSACK_CI_NODE_TOTAL: ${{ matrix.ci_node_total }}
+          KNAPSACK_CI_NODE_INDEX: ${{ matrix.ci_node_index }}
+          PARALLEL_TESTS_CONCURRENCY: ${{ env.cpu_cores }}
         run: |
-          bundle exec rspec
+          bundle exec parallel_rspec -n $PARALLEL_TESTS_CONCURRENCY -e './bin/parallel_tests'
       - name: Check for missing annotations
         run: bundle exec annotate
       - name: Check for untracked changes in app and spec directories
@@ -58,4 +87,27 @@ jobs:
           path: "./app/ ./spec/"
       - name: Report to CodeClimate
         run: |
-          ./cc-test-reporter after-build --exit-code 0
+          ./cc-test-reporter format-coverage --output "coverage/coverage.${{ matrix.ci_node_index }}.json"
+      - name: Upload partial converage
+        uses: actions/upload-artifact@v2
+        with:
+          name: coverage
+          path: "coverage/coverage.${{ matrix.ci_node_index }}.json"
+  coverage:
+    name: Coverage
+    runs-on: ubuntu-latest
+    needs: tests
+    steps:
+      - name: Setup Code Climate test-reporter
+        run: |
+          curl -L https://codeclimate.com/downloads/test-reporter/test-reporter-latest-linux-amd64 > ./cc-test-reporter
+          chmod +x ./cc-test-reporter
+      - name: download coverage reports
+        uses: actions/download-artifact@v2
+        with:
+          name: coverage
+          path: coverage/coverage.*.json
+      - name: report coverage
+        run: |
+          ./cc-test-reporter sum-coverage coverage/**/*.json
+          ./cc-test-reporter upload-coverage

.github/workflows/update_knapsack_report.yml

@@ -0,0 +1,63 @@
+name: Update Knapsack Report
+
+# This workflow should be scheduled at certain intervals
+on:
+ schedule:
+     - cron: '0 5 31 2 *'    
+# The above cron does not run. Replace with the wanted periodicity.
+# For example, the following configuration would schedule it every 3 months
+#    - cron: '0 0 1 */3 *'
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+
+    services:
+      db:
+        image: postgres:15
+        env:
+          POSTGRES_USER: postgres
+          POSTGRES_HOST_AUTH_METHOD: trust
+        ports:
+          - 5432:5432
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+
+    env:
+      RAILS_ENV: test
+
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up Ruby 3.1
+      uses: actions/setup-ruby@v1
+      with:
+        ruby-version: 3.1
+        github_token: ${{ secrets.GITHUB_TOKEN }}
+    - name: Setup Database
+      run: bundle exec rails db:create db:migrate
+    - name: Update Knapsack Report
+      run: KNAPSACK_GENERATE_REPORT=true bundle exec rspec
+    - name: Commit files
+      run: |
+        git config --local user.email "action@github.com"
+        git config --local user.name "GitHub Action"
+        git add knapsack_rspec_report.json
+        git commit -m "Update knapsack_rspec_report.json"
+    - name: Push changes
+      run: git push origin HEAD:update-knapsack-report
+    - name: Initialize Pull Request
+      uses: rootstrap/create-pull-request@v3
+      with:
+        pull_request_token: ${{ secrets.GITHUB_TOKEN }}
+        head: update-knapsack-report
+        base: main
+        title: 'Update Knapsack report'
+        maintainer_can_modify: true
+        draft: false
+        body: >
+          This is an automated PR. Knapsack uses the report file to distribute tests across different nodes
+          so that they all have similar execution times. It's advisable to update the report periodically, 
+          especially if many test files have been added, for better distribution.

Gemfile

@@ -41,6 +41,8 @@ group :development, :test do
   gem 'annotate', '~> 3.2', '>= 3.0.3'
   gem 'dotenv-rails', '~> 2.7.6'
   gem 'factory_bot_rails', '~> 6.2'
+  gem 'knapsack', '~> 4.0'
+  gem 'parallel_tests', '~> 4.2'
   gem 'pry-byebug', '~> 3.9', platform: :mri
   gem 'pry-rails', '~> 0.3.9'
   gem 'rspec_api_documentation', '~> 6.1.0'

Gemfile.lock

@@ -118,6 +118,7 @@ GEM
       aws-sigv4 (~> 1.6)
     aws-sigv4 (1.6.0)
       aws-eventstream (~> 1, >= 1.0.2)
+    base64 (0.1.1)
     bcrypt (3.1.18)
     better_errors (2.10.1)
       erubi (>= 1.0.0)
@@ -248,6 +249,8 @@ GEM
       activerecord
       kaminari-core (= 1.2.2)
     kaminari-core (1.2.2)
+    knapsack (4.0.0)
+      rake
     kwalify (0.7.2)
     language_server-protocol (3.17.0.3)
     launchy (2.5.0)
@@ -293,7 +296,9 @@ GEM
     oj (3.16.1)
     orm_adapter (0.5.0)
     pagy (4.11.0)
-    parallel (1.23.0)
+    parallel (1.22.1)
+    parallel_tests (4.2.1)
+      parallel
     parser (3.2.2.3)
       ast (~> 2.4.1)
       racc
@@ -408,7 +413,8 @@ GEM
       activesupport (>= 3.0.0)
       mustache (~> 1.0, >= 0.99.4)
       rspec (~> 3.0)
-    rubocop (1.55.0)
+    rubocop (1.56.0)
+      base64 (~> 0.1.1)
       json (~> 2.3)
       language_server-protocol (>= 3.17.0)
       parallel (~> 1.10)
@@ -524,11 +530,13 @@ DEPENDENCIES
   flipper-ui (~> 0.28.0)
   i18n-tasks (~> 1.0.12)
   jbuilder (~> 2.10)
+  knapsack (~> 4.0)
   letter_opener (~> 1.7)
   listen (~> 3.8)
   lograge (~> 0.13)
   oj (~> 3.16)
   pagy (~> 4.0)
+  parallel_tests (~> 4.2)
   pg (~> 1.5)
   pg_query (~> 4.2.1)
   prosopite (~> 1.3.2)

README.md

@@ -81,9 +81,11 @@ To illustrate, `bin/rails console` will run the console in the docker container
 - [Faker](https://github.com/stympy/faker) for generating test data
 - [Flipper](https://github.com/jnunemaker/flipper) for feature flag support
 - [Jbuilder](https://github.com/rails/jbuilder) for json views
+- [Knapsack](https://github.com/KnapsackPro/knapsack) for generating CI time report and splitting into nodes
 - [Letter Opener](https://github.com/ryanb/letter_opener) for previewing a mail in the browser
 - [Oj](https://github.com/ohler55/oj) for optimized json
 - [Pagy](https://github.com/ddnexus/pagy) for pagination
+- [Parallel Tests](https://github.com/grosser/parallel_tests) for split CI in several cores
 - [Prosopite](https://github.com/charkost/prosopite) to detect N+1 queries
 - [Pry](https://github.com/pry/pry) for enhancing the ruby shell
 - [Puma](https://github.com/puma/puma) for the server
@@ -105,6 +107,7 @@ To illustrate, `bin/rails console` will run the console in the docker container
 - Set your [frontend URL](https://github.com/cyu/rack-cors#origin) in `config/initializers/rack_cors.rb`
 - Set your mail sender in `config/initializers/devise.rb`
 - Config your timezone accordingly in `application.rb`.
+- Config CI in parallel execution. See [docs](docs/ci.md).
 
 ## Api Docs
 

Rakefile

@@ -6,3 +6,5 @@
 require_relative 'config/application'
 
 Rails.application.load_tasks
+
+Knapsack.load_tasks if defined?(Knapsack)

bin/parallel_tests

@@ -0,0 +1,18 @@
+#!/bin/bash
+# This file should be in bin/parallel_tests
+
+# updates CI node total based on parallel_tests concurrency
+KNAPSACK_CI_NODE_TOTAL=$(( $PARALLEL_TESTS_CONCURRENCY * $KNAPSACK_CI_NODE_TOTAL ))
+
+if [ "$TEST_ENV_NUMBER" == "" ]; then
+  PARALLEL_TESTS_CONCURRENCY_INDEX=0
+else
+  PARALLEL_TESTS_CONCURRENCY_INDEX=$(( $TEST_ENV_NUMBER - 1 ))
+fi
+
+KNAPSACK_CI_NODE_INDEX=$(( $PARALLEL_TESTS_CONCURRENCY_INDEX + ($PARALLEL_TESTS_CONCURRENCY * $KNAPSACK_CI_NODE_INDEX) ))
+
+# logs info about ENVs to ensure everything works
+echo KNAPSACK_CI_NODE_TOTAL=$KNAPSACK_CI_NODE_TOTAL KNAPSACK_CI_NODE_INDEX=$KNAPSACK_CI_NODE_INDEX PARALLEL_TESTS_CONCURRENCY=$PARALLEL_TESTS_CONCURRENCY
+
+CI_NODE_TOTAL=$KNAPSACK_CI_NODE_TOTAL CI_NODE_INDEX=$KNAPSACK_CI_NODE_INDEX bundle exec rake knapsack:rspec

config/database.yml

@@ -13,7 +13,7 @@ development:
 
 test:
   <<: *default
-  database: rails_api_base_test
+  database: rails_api_base_test<%= ENV['TEST_ENV_NUMBER'] %>
 
 # For production is recommended to set DATABASE_URL env variable.
 # It will take precedence over the config defined here.

docs/ci.md

@@ -0,0 +1,44 @@
+# CI
+
+## Parallelization with Parallel Tests & Knapsack
+Knapsack and Parallel Tests gems allow us to run tests in several nodes at the same time, benefiting us in the execution time. Knapsack parallelizes them at node level while Parallel Tests does it at CPU level.
+
+Knapsack splits tests based on an execution time report. In case there are files that were not added in the report, they will all run on the same node and may overload it, so it is strongly recommended to update the report frequently.
+
+## Configuration
+In case you want to use this you will need the script that splits spec files called `parallel_tests`, which sets up the configuration, assuming you have `n_nodes * cpu_cores_quantity`.
+
+On Github Actions you can add any nodes you want using matrix strategy, setting up some variables:
+
+```sh
+  ci_node_total: [4]
+  # set N-1 indexes for parallel jobs
+  # When you run 2 parallel jobs then first job will have index 0, the second job will have index 1 etc
+  ci_node_index: [0, 1, 2, 3]
+```
+
+CPU cores quantity on every node are obtained automatically from Github Actions config `echo "cpu_cores=$(nproc)" >> $GITHUB_ENV`.
+
+If you want to update it manually you can do it by updating this variable:
+`PARALLEL_TESTS_CONCURRENCY: 2`
+
+To update tests on local machine you can execute `KNAPSACK_CI_NODE_TOTAL=4 KNAPSACK_CI_NODE_INDEX=1 PARALLEL_TESTS_CONCURRENCY=2 bundle exec parallel_rspec -n 2 -e './bin/parallel_tests'`. This will run subset of tests files corresponding to second node.
+
+## Generating report
+Knapsack report needs to be updated frequently to balance execution time among nodes. This can be done manually by executing:
+`KNAPSACK_GENERATE_REPORT=true bundle exec rspec`
+
+It is also recommended to generate the report in the CI for a better precision. For this you have available a workflow in Github Actions that triggers the report generation and creates a pull request automatically. This workflow can be scheduled in the frequency you want or even can be manually triggered.
+
+To schedule the cron task you have to do it in `.github/workflows/update_knapsack_report.yml:6`
+It is now scheduled for February 31 so will never run.
+
+```sh
+  - cron: '0 5 31 2 *'    
+  # The above cron does not run. Replace with the wanted periodicity.
+```
+## Coverage
+When splitting tests in different nodes, each report covers only a part of the code files being tested.
+For this reason a job in the CI is added to sums coverages from all nodes to be used by SimpleCov. This job will be executed after all nodes have finished and will send the final report to CodeClimate.
+
+For the case of CPU cores we do not need to add extra configuration since the report of each node contains the info of all the cores that have been splited.

knapsack_rspec_report.json

@@ -0,0 +1,31 @@
+{
+  "spec/policies/admin/page_policy_spec.rb": 0.038240999972913414,
+  "spec/routing/sessions_routing_spec.rb": 0.006703999999444932,
+  "spec/requests/api/v1/users/create_spec.rb": 0.5093869999982417,
+  "spec/acceptance/settings_spec.rb": 0.17437299998709932,
+  "spec/acceptance/sessions_spec.rb": 0.5090029999846593,
+  "spec/acceptance/status_spec.rb": 0.009465000010095537,
+  "spec/requests/api/v1/sessions/destroy_spec.rb": 0.2923869999940507,
+  "spec/policies/admin/user_policy_spec.rb": 0.020087999990209937,
+  "spec/routing/registrations_routing_spec.rb": 0.001361000002361834,
+  "spec/policies/admin/application_policy_spec.rb": 0.011096000031102449,
+  "spec/policies/admin_user_policy_spec.rb": 0.009194999991450459,
+  "spec/routing/user_routing_spec.rb": 0.0026309999520890415,
+  "spec/requests/api/v1/passwords/update_spec.rb": 0.4883949999930337,
+  "spec/requests/api/v1/feature_flags_spec.rb": 0.040616000012960285,
+  "spec/requests/api/v1/settings_spec.rb": 0.0425929999910295,
+  "spec/requests/api/v1/passwords/create_spec.rb": 0.15221400000154972,
+  "spec/acceptance/users_spec.rb": 0.5729850000352599,
+  "spec/policies/user_policy_spec.rb": 0.015309000038541853,
+  "spec/requests/api/v1/users/update_spec.rb": 0.9842829999979585,
+  "spec/requests/api/v1/status_spec.rb": 0.01666500000283122,
+  "spec/policies/application_policy_spec.rb": 0.008917000028304756,
+  "spec/policies/admin/admin_user_policy_spec.rb": 0.005394000036176294,
+  "spec/requests/api/v1/sessions/create_spec.rb": 0.49404099996900186,
+  "spec/decorators/user_decorator_spec.rb": 0.005971000005956739,
+  "spec/requests/api/v1/passwords/edit_spec.rb": 0.08089700003620237,
+  "spec/acceptance/passwords_spec.rb": 0.6197870000032708,
+  "spec/models/user_spec.rb": 0.09392800001660362,
+  "spec/mailers/application_mailer_spec.rb": 0.0020680000307038426,
+  "spec/requests/api/v1/users/show_spec.rb": 0.5047900000354275
+}

lib/tasks/code_analysis.rake

@@ -7,5 +7,6 @@ namespace :code do
     sh 'bundle exec rubocop .'
     sh 'bundle exec reek .'
     sh 'bundle exec rails_best_practices .'
+    sh 'bundle exec i18n-tasks health'
   end
 end

spec/rails_helper.rb

@@ -11,6 +11,7 @@
   add_filter 'app/admin'
   add_filter 'config'
   add_filter 'spec'
+  add_filter 'lib/tasks/code_analysis.rake'
 end
 
 require File.expand_path('../config/environment', __dir__)