Circle flags

deployment.rb

@@ -364,3 +364,7 @@ def pegasus_dir(*paths)
 def shared_dir(*dirs)
   deploy_dir('shared', *dirs)
 end
+
+def lib_dir(*dirs)
+  deploy_dir('lib', *dirs)
+end

lib/Rakefile

@@ -0,0 +1,12 @@
+require 'rake/testtask'
+
+desc 'Run tests for lib directory'
+task :test do
+  puts 'starting test task'
+  Rake::TestTask.new do |t|
+    t.warning = false
+    t.test_files = FileList["test/**/test_*.rb"]
+  end
+end
+
+task :default => :test

lib/cdo/circle_utils.rb

@@ -0,0 +1,32 @@
+require 'set'
+
+module CircleUtils
+  # Checks the HEAD commit for the current circle build for the specified tag,
+  # returning TRUE if it's present.  A tag is a set of space-separated words
+  # wrapped in square brackets.  The words can be given in any order.
+  #
+  # Example:
+  #   CircleUtils.tagged?('skip ui') will match commit messages containing the
+  #   strings "[skip ui]" or "[ui skip]"
+  def self.tagged?(tag)
+    build_tags.include?(tag.downcase.split.to_set)
+  end
+
+  def self.circle_commit_message
+    `git log --format=%B -n 1 $CIRCLE_SHA1`.strip
+  end
+
+  # @return [Set<Set<String>>] set of build tags in this build's commit message
+  private_class_method def self.build_tags
+    # Only parse the commit message once
+    @tags ||= circle_commit_message.
+        scan(/(?<=\[)[\w\d\s]+(?=\])/).
+        map {|s| s.downcase.split.to_set }.
+        to_set
+  end
+
+  # In unit tests, we want to bypass the cache and recompute tags.
+  def self.__clear_cached_tags_for_test
+    @tags = nil
+  end
+end

lib/cdo/git_utils.rb

@@ -36,18 +36,10 @@ def self.files_changed_locally
     staged_changes.concat(unstaged_changes).uniq
   end
 
-  def self.circle_commit_contains?(string)
-    circle_commit_message.include?(string)
-  end
-
   def self.current_branch
     `git rev-parse --abbrev-ref HEAD`.strip
   end
 
-  def self.circle_commit_message
-    `git log --format=%B -n 1 $CIRCLE_SHA1`.strip
-  end
-
   def self.get_latest_commit_merged_branch
     get_branch_commit_merges(git_revision)
   end

lib/cdo/test_run_utils.rb

@@ -66,4 +66,10 @@ def self.run_shared_tests
       end
     end
   end
+
+  def self.run_lib_tests
+    Dir.chdir(lib_dir) do
+      RakeUtils.rake_stream_output 'test'
+    end
+  end
 end

lib/rake/circle.rake

@@ -1,17 +1,48 @@
 require_relative '../../deployment'
 require 'cdo/rake_utils'
+require 'cdo/circle_utils'
 require 'cdo/git_utils'
 require 'open-uri'
 require 'json'
 
-RUN_ALL_TESTS_TAG = '[test all]'
-SKIP_UI_TESTS_TAG = '[skip ui]'
+# CircleCI Build Tags
+# We provide some limited control over CircleCI's build behavior by adding these
+# tags to the latest commit message.  A tag is a set of words in [] square
+# brackets - those words can be in any order and are case-insensitive.
+#
+# Supported Tags:
+
+# Don't run Circle at all (built-in to CircleCI)
+# 'ci skip'
+
+# Run all unit/integration tests, not just a subset based on changed files.
+RUN_ALL_TESTS_TAG = 'test all'.freeze
+
+# Don't run any UI or Eyes tests.
+SKIP_UI_TESTS_TAG = 'skip ui'.freeze
+
+# Don't run UI tests against Chrome
+SKIP_CHROME_TAG = 'skip chrome'.freeze
+
+# Run UI tests against Firefox45Win7
+TEST_FIREFOX_TAG = 'test firefox'.freeze
+
+# Run UI tests against IE11Win10
+TEST_IE_TAG = 'test ie'.freeze
+TEST_IE_VERBOSE_TAG = 'test internet explorer'.freeze
+
+# Run UI tests against SafariYosemite
+TEST_SAFARI_TAG = 'test safari'.freeze
+
+# Overrides for whether to run Applitools eyes tests
+TEST_EYES = 'test eyes'.freeze
+SKIP_EYES = 'skip eyes'.freeze
 
 namespace :circle do
   desc 'Runs tests for changed sub-folders, or all tests if the tag specified is present in the most recent commit message.'
   task :run_tests do
-    if GitUtils.circle_commit_contains?(RUN_ALL_TESTS_TAG)
-      HipChat.log "Commit message: '#{GitUtils.circle_commit_message}' contains #{RUN_ALL_TESTS_TAG}, force-running all tests."
+    if CircleUtils.tagged?(RUN_ALL_TESTS_TAG)
+      HipChat.log "Commit message: '#{CircleUtils.circle_commit_message}' contains [#{RUN_ALL_TESTS_TAG}], force-running all tests."
       RakeUtils.rake_stream_output 'test:all'
     else
       RakeUtils.rake_stream_output 'test:changed'
@@ -20,35 +51,31 @@ namespace :circle do
 
   desc 'Runs UI tests only if the tag specified is present in the most recent commit message.'
   task :run_ui_tests do
-    if GitUtils.circle_commit_contains?(SKIP_UI_TESTS_TAG)
-      HipChat.log "Commit message: '#{GitUtils.circle_commit_message}' contains #{SKIP_UI_TESTS_TAG}, skipping UI tests for this run."
+    if CircleUtils.tagged?(SKIP_UI_TESTS_TAG)
+      HipChat.log "Commit message: '#{CircleUtils.circle_commit_message}' contains [#{SKIP_UI_TESTS_TAG}], skipping UI tests for this run."
       next
     end
     RakeUtils.exec_in_background 'RACK_ENV=test RAILS_ENV=test bundle exec ./bin/dashboard-server'
-    RakeUtils.system_stream_output 'wget https://saucelabs.com/downloads/sc-4.4.0-rc2-linux.tar.gz'
-    RakeUtils.system_stream_output 'tar -xzf sc-4.4.0-rc2-linux.tar.gz'
-    Dir.chdir(Dir.glob('sc-*-linux')[0]) do
-      # Run sauce connect a second time on failure, known periodic "Error bringing up tunnel VM." disconnection-after-connect issue, e.g. https://circleci.com/gh/code-dot-org/code-dot-org/20930
-      RakeUtils.exec_in_background "for i in 1 2; do ./bin/sc -vv -l $CIRCLE_ARTIFACTS/sc.log -u $SAUCE_USERNAME -k $SAUCE_ACCESS_KEY -i #{CDO.circle_run_identifier} --tunnel-domains localhost-studio.code.org,localhost.code.org && break; done"
-    end
+    start_sauce_connect
     RakeUtils.system_stream_output 'until $(curl --output /dev/null --silent --head --fail http://localhost.studio.code.org:3000); do sleep 5; done'
     Dir.chdir('dashboard/test/ui') do
-      is_pipeline_branch = ['staging', 'test', 'production'].include?(GitUtils.current_branch)
       container_features = `find ./features -name '*.feature' | sort | awk "NR % (${CIRCLE_NODE_TOTAL} - 1) == (${CIRCLE_NODE_INDEX} - 1)"`.split("\n").map{|f| f[2..-1]}
       eyes_features = `grep -lr '@eyes' features`.split("\n")
       container_eyes_features = container_features & eyes_features
-      browsers_to_run = 'ChromeLatestWin7'
-      RakeUtils.system_stream_output "bundle exec ./runner.rb" \
-          " --feature #{container_features.join(',')}" \
-          " --config #{browsers_to_run}" \
-          " --pegasus localhost.code.org:3000" \
-          " --dashboard localhost.studio.code.org:3000" \
-          " --circle" \
-          " --parallel 16" \
-          " --abort_when_failures_exceed 10" \
-          " --retry_count 2" \
-          " --html"
-      if is_pipeline_branch
+      ui_test_browsers = browsers_to_run
+      unless ui_test_browsers.empty?
+        RakeUtils.system_stream_output "bundle exec ./runner.rb" \
+            " --feature #{container_features.join(',')}" \
+            " --config #{ui_test_browsers.join(',')}" \
+            " --pegasus localhost.code.org:3000" \
+            " --dashboard localhost.studio.code.org:3000" \
+            " --circle" \
+            " --parallel 16" \
+            " --abort_when_failures_exceed 10" \
+            " --retry_count 2" \
+            " --html"
+      end
+      if test_eyes?
         RakeUtils.system_stream_output "bundle exec ./runner.rb" \
             " --eyes" \
             " --feature #{container_eyes_features.join(',')}" \
@@ -61,8 +88,39 @@ namespace :circle do
             " --html"
       end
     end
-    # Kill Sauce Connect tunnel
-    RakeUtils.system_stream_output 'killall sc'
+    close_sauce_connect
     RakeUtils.system_stream_output 'sleep 10'
   end
 end
+
+def pipeline_branch?
+  ['staging', 'test', 'production'].include?(GitUtils.current_branch)
+end
+
+# @return [Array<String>] names of browser configurations for this test run
+def browsers_to_run
+  browsers = []
+  browsers << 'ChromeLatestWin7' unless CircleUtils.tagged?(SKIP_CHROME_TAG)
+  browsers << 'Firefox45Win7' if CircleUtils.tagged?(TEST_FIREFOX_TAG)
+  browsers << 'IE11Win10' if CircleUtils.tagged?(TEST_IE_TAG) || CircleUtils.tagged?(TEST_IE_VERBOSE_TAG)
+  browsers << 'SafariYosemite' if CircleUtils.tagged?(TEST_SAFARI_TAG)
+  browsers
+end
+
+def test_eyes?
+  !CircleUtils.tagged?(SKIP_EYES) &&
+      (pipeline_branch? || CircleUtils.tagged?(TEST_EYES))
+end
+
+def start_sauce_connect
+  RakeUtils.system_stream_output 'wget https://saucelabs.com/downloads/sc-4.4.0-rc2-linux.tar.gz'
+  RakeUtils.system_stream_output 'tar -xzf sc-4.4.0-rc2-linux.tar.gz'
+  Dir.chdir(Dir.glob('sc-*-linux')[0]) do
+    # Run sauce connect a second time on failure, known periodic "Error bringing up tunnel VM." disconnection-after-connect issue, e.g. https://circleci.com/gh/code-dot-org/code-dot-org/20930
+    RakeUtils.exec_in_background "for i in 1 2; do ./bin/sc -vv -l $CIRCLE_ARTIFACTS/sc.log -u $SAUCE_USERNAME -k $SAUCE_ACCESS_KEY -i #{CDO.circle_run_identifier} --tunnel-domains localhost-studio.code.org,localhost.code.org && break; done"
+  end
+end
+
+def close_sauce_connect
+  RakeUtils.system_stream_output 'killall sc'
+end

lib/rake/test.rake

@@ -31,6 +31,11 @@ namespace :test do
     TestRunUtils.run_shared_tests
   end
 
+  desc 'Runs lib tests.'
+  task :lib do
+    TestRunUtils.run_lib_tests
+  end
+
   namespace :changed do
     desc 'Runs apps tests if apps might have changed from staging.'
     task :apps do
@@ -60,12 +65,19 @@ namespace :test do
       end
     end
 
-    task all: [:apps, :dashboard, :pegasus, :shared]
+    desc 'Runs lib tests if lib might have changed from staging.'
+    task :lib do
+      run_tests_if_changed('lib', ['lib/**/*']) do
+        TestRunUtils.run_lib_tests
+      end
+    end
+
+    task all: [:apps, :dashboard, :pegasus, :shared, :lib]
   end
 
   task changed: ['changed:all']
 
-  task all: [:apps, :dashboard, :pegasus, :shared]
+  task all: [:apps, :dashboard, :pegasus, :shared, :lib]
 end
 task test: ['test:changed']
 

lib/test/cdo/test_circle_utils.rb

@@ -0,0 +1,55 @@
+require_relative '../../../shared/test/test_helper'
+require 'cdo/circle_utils'
+
+class CircleUtilsTest < Minitest::Test
+  def teardown
+    CircleUtils.__clear_cached_tags_for_test
+  end
+
+  def test_knows_when_tag_is_present
+    CircleUtils.stub :circle_commit_message, 'message [foo]' do
+      assert CircleUtils.tagged? 'foo'
+      refute CircleUtils.tagged? 'bar'
+    end
+  end
+
+  def test_tags_are_case_insensitive
+    CircleUtils.stub :circle_commit_message, 'message [Foo]' do
+      assert CircleUtils.tagged? 'foo'
+      assert CircleUtils.tagged? 'Foo'
+      assert CircleUtils.tagged? 'FOO'
+    end
+  end
+
+  def test_does_not_see_commit_message_as_a_tag
+    CircleUtils.stub :circle_commit_message, 'message [foo] suffix' do
+      refute CircleUtils.tagged? 'message'
+      refute CircleUtils.tagged? 'suffix'
+    end
+  end
+
+  def test_sees_multiple_tags
+    CircleUtils.stub :circle_commit_message, 'message [foo] [bar]' do
+      assert CircleUtils.tagged? 'foo'
+      assert CircleUtils.tagged? 'bar'
+      refute CircleUtils.tagged? 'baz'
+    end
+  end
+
+  def test_multi_word_tags
+    CircleUtils.stub :circle_commit_message, 'message [foo bar]' do
+      # Detects correct word combination
+      assert CircleUtils.tagged? 'foo bar'
+      refute CircleUtils.tagged? 'bar baz'
+
+      # Is whitespace-agnostic
+      assert CircleUtils.tagged? 'foo   bar'
+
+      # Is word-order agnostic
+      assert CircleUtils.tagged? 'bar foo'
+
+      # Ignores repeated words (a tag is a Set)
+      assert CircleUtils.tagged? 'foo foo bar bar bar'
+    end
+  end
+end

lib/test/cdo/test_properties.rb

@@ -45,5 +45,6 @@ def test_delete_when_key_does_not_exist
 
   def teardown
     DB[:properties].where(key: KEY).delete
+    (1..5).each {|i| DB[:properties].where(key: "key#{i}").delete}
   end
 end