Automatically detect and track flaky tests

.github/workflows/tests.yml

@@ -320,3 +320,14 @@ jobs:
       - run: brew test-bot --only-formulae --test-default-formula
 
       - uses: codecov/codecov-action@29386c70ef20e286228c72b668a06fd0e8399192
+
+      - name: Upload test results to BuildPulse for flaky test detection
+        # Only run this step for PRs where where we have access to secrets.
+        # Run this step even when the tests fail. Skip if the workflow is cancelled.
+        if: github.event.pull_request.head.repo.full_name == github.repository && !cancelled()
+        env:
+          BUILDPULSE_ACCESS_KEY_ID: ${{ secrets.BUILDPULSE_ACCESS_KEY_ID }}
+          BUILDPULSE_SECRET_ACCESS_KEY: ${{ secrets.BUILDPULSE_SECRET_ACCESS_KEY }}
+        run: |
+          brew install buildpulse-test-reporter
+          buildpulse-test-reporter submit Library/Homebrew/test/junit --account-id 1503512 --repository-id 53238813

.gitignore

@@ -14,6 +14,7 @@
 /Library/Homebrew/test/.gem
 /Library/Homebrew/test/.subversion
 /Library/Homebrew/test/coverage
+/Library/Homebrew/test/junit
 /Library/Homebrew/test/fs_leak_log
 /Library/Homebrew/vendor/portable-ruby
 /Library/Taps
@@ -133,6 +134,7 @@
 **/vendor/bundle/ruby/*/gems/rspec-*/
 **/vendor/bundle/ruby/*/gems/rspec-core-*/
 **/vendor/bundle/ruby/*/gems/rspec-expectations-*/
+**/vendor/bundle/ruby/*/gems/rspec_junit_formatter-*/
 **/vendor/bundle/ruby/*/gems/rspec-its-*/
 **/vendor/bundle/ruby/*/gems/rspec-mocks-*/
 **/vendor/bundle/ruby/*/gems/rspec-retry-*/

Library/Homebrew/.rspec_parallel

@@ -0,0 +1,6 @@
+--format NoSeedProgressFormatter
+--format ParallelTests::RSpec::RuntimeLogger
+--out <%= ENV["PARALLEL_RSPEC_LOG_PATH"] %>
+--format RspecJunitFormatter
+--out test/junit/rspec<%= ENV["TEST_ENV_NUMBER"] %>.xml
+<%= "--format RSpec::Github::Formatter" if ENV["GITHUB_ACTIONS"] %>

Library/Homebrew/Gemfile

@@ -12,6 +12,7 @@ gem "ronn", require: false
 gem "rspec", require: false
 gem "rspec-github", require: false
 gem "rspec-its", require: false
+gem "rspec_junit_formatter", require: false
 gem "rspec-retry", require: false
 gem "rspec-wait", require: false
 gem "rubocop", require: false

Library/Homebrew/Gemfile.lock

@@ -109,6 +109,8 @@ GEM
     rspec-support (3.10.2)
     rspec-wait (0.0.9)
       rspec (>= 3, < 4)
+    rspec_junit_formatter (0.4.1)
+      rspec-core (>= 2, < 4, != 2.12.0)
     rubocop (1.17.0)
       parallel (~> 1.10)
       parser (>= 3.0.0.0)
@@ -194,6 +196,7 @@ DEPENDENCIES
   rspec-retry
   rspec-sorbet
   rspec-wait
+  rspec_junit_formatter
   rubocop
   rubocop-ast
   rubocop-performance

Library/Homebrew/dev-cmd/tests.rb

@@ -111,6 +111,7 @@ def tests
       else
         "#{HOMEBREW_CACHE}/#{parallel_rspec_log_name}"
       end
+      ENV["PARALLEL_RSPEC_LOG_PATH"] = parallel_rspec_log_path
 
       parallel_args = if ENV["CI"]
         %W[
@@ -133,13 +134,8 @@ def tests
         --seed #{seed}
         --color
         --require spec_helper
-        --format NoSeedProgressFormatter
-        --format ParallelTests::RSpec::RuntimeLogger
-        --out #{parallel_rspec_log_path}
       ]
 
-      bundle_args << "--format" << "RSpec::Github::Formatter" if ENV["GITHUB_ACTIONS"]
-
       unless OS.mac?
         bundle_args << "--tag" << "~needs_macos" << "--tag" << "~cask"
         files = files.reject { |p| p =~ %r{^test/(os/mac|cask)(/.*|_spec\.rb)$} }