Refactor to run one workflow per-environment

.circleci/config.yml

@@ -1,39 +1,114 @@
-version: 2
-jobs:
-  test:
+---
+version: 2.1
+
+executors:
+  # Common container definition used by all jobs
+  ruby_browsers:
     docker:
       - image: circleci/ruby:2.5.5-browsers
+
+commands:
+  # Smoke test command template
+  smoketest:
+    parameters:
+      env:
+        type: string
     steps:
       - checkout
       - run:
           name: Setup bundler
           command: gem install bundler
-      - restore-cache:
+      - restore_cache:
           keys:
             - v2-identity-monitor-bundle-{{ checksum "Gemfile.lock" }}
       - run:
           name: Install dependencies
           command: |
             bundle install --path vendor/bundle
-      - save-cache:
+      - save_cache:
           key: v2-identity-monitor-bundle-{{ checksum "Gemfile.lock" }}
           paths:
             - vendor/bundle
       - run:
           name: Setup tests
           command: bin/circleci-setup
       - run:
-          name: Run tests
-          command: bin/circleci-run
+          name: Run smoketest suite against environment
+          command: bin/circleci-run << parameters.env >>
+
+jobs:
+  smoketest_dev:
+    executor: ruby_browsers
+    steps:
+      - smoketest:
+          env: dev
+
+  smoketest_int:
+    executor: ruby_browsers
+    steps:
+      - smoketest:
+          env: int
+
+  smoketest_staging:
+    executor: ruby_browsers
+    steps:
+      - smoketest:
+          env: staging
+
+  smoketest_prod:
+    executor: ruby_browsers
+    steps:
+      - smoketest:
+          env: prod
+
 workflows:
-   version: 2
-   hourly:
-     triggers:
-       - schedule:
-           cron: "0 * * * *"
-           filters:
-             branches:
-               only:
-                 - master
-     jobs:
-       - test
+  # DEV - Not yet ready
+  # smoketest_dev:
+  #   triggers:
+  #     - schedule:
+  #         cron: "20 * * * *"
+  #         filters:
+  #           branches:
+  #             only:
+  #               - master
+  #   jobs:
+  #     - prepare
+  #     - smoketest_dev:
+  #       requires:
+  #         - prepare
+
+  # INT
+  smoketest_int:
+    triggers:
+      - schedule:
+          cron: "30 * * * *"
+          filters:
+            branches:
+              only:
+                - master
+    jobs:
+      - smoketest_int
+
+  # STAGING
+  smoketest_staging:
+    triggers:
+      - schedule:
+          cron: "40 * * * *"
+          filters:
+            branches:
+              only:
+                - master
+    jobs:
+      - smoketest_staging
+
+  # PROD
+  smoketest_prod:
+    triggers:
+      - schedule:
+          cron: "50 * * * *"
+          filters:
+            branches:
+              only:
+                - master
+    jobs:
+      - smoketest_prod

.rubocop.yml

@@ -5,7 +5,7 @@
 # https://github.com/bbatsov/rubocop/blob/master/config/disabled.yml
 
 AllCops:
-  TargetRubyVersion: 2.3
+  TargetRubyVersion: 2.5
   UseCache: true
   Exclude:
     - 'Gemfile'

bin/circleci-run

@@ -1,25 +1,58 @@
 #!/usr/bin/env ruby
 
-##
-# There are lots of moving parts so the tests can be flaky.
-# For better or worse we only sound the alarm if we fail after 3 tries.
-#
-def run_and_retry_tests(environment)
-  result = system "HEADLESS_BROWSER=true LOWER_ENV=#{environment} bundle exec rspec"
-  return if result
-
-  2.times do
+require 'optparse'
+
+def run_and_retry_tests(environment, retries)
+  cmd = ['HEADLESS_BROWSER=true', "LOWER_ENV=#{environment}", 'bundle', 'exec',
+         'rspec']
+  retry_flags = ['--only-failure']
+
+  retries.times do
+    return if system cmd.join(' ')
+
     puts "Smoke tests failed in #{environment}. Retrying..."
-    return if system "HEADLESS_BROWSER=true LOWER_ENV=#{environment} bundle exec rspec --only-failure"
+
+    # Retries use '--only-failure' to skip over previous sucessful tests
+    retry_flags.empty? or cmd << retry_flags.shift
   end
   raise "Tests failed in #{environment}"
 end
 
-puts 'Running tests in INT'
-run_and_retry_tests('int')
+def main
+  basename = File.basename($PROGRAM_NAME)
+  retries = 3
+
+  optparse = OptionParser.new do |opts|
+    opts.banner = <<~USAGEDOC
+      usage: #{basename} [-r NUM] ENVIRONMENT
+
+      Run smoke tests for a given environment.  (Intended for pipeline use.)
+
+      Args:
+          ENVIRONMENT Target to test.  Sets "LOWER_ENV" in the rspec test.
+                      Environment variables for the target must be present.
+
+    USAGEDOC
 
-puts 'Running tests in STAGING'
-run_and_retry_tests('staging')
+    opts.on('-r', '--retry-count NUM', Integer,
+            'Times to retry test.  (Default: 3)') do |r|
+      retries = r
+    end
+  end
+
+  args = optparse.parse!
+
+  if args.length != 1
+    warn optparse
+    exit 1
+  end
 
-puts 'Renning tests in PROD'
-run_and_retry_tests('prod')
+  env = ARGV[0]
+
+  puts "Running tests in #{env}"
+  run_and_retry_tests(env, retries)
+end
+
+if __FILE__ == $PROGRAM_NAME
+  main
+end