Record flaky tests

ruby/lib/ci/queue/configuration.rb

@@ -2,7 +2,7 @@
 module CI
   module Queue
     class Configuration
-      attr_accessor :timeout, :worker_id, :max_requeues, :grind_count, :failure_file
+      attr_accessor :timeout, :worker_id, :max_requeues, :grind_count, :failure_file, :export_flaky_tests_file
       attr_accessor :requeue_tolerance, :namespace, :failing_test, :statsd_endpoint
       attr_accessor :max_test_duration, :max_test_duration_percentile, :track_test_duration
       attr_accessor :max_test_failed, :redis_ttl
@@ -35,7 +35,8 @@ def initialize(
         namespace: nil, seed: nil, flaky_tests: [], statsd_endpoint: nil, max_consecutive_failures: nil,
         grind_count: nil, max_duration: nil, failure_file: nil, max_test_duration: nil,
         max_test_duration_percentile: 0.5, track_test_duration: false, max_test_failed: nil,
-        queue_init_timeout: nil, redis_ttl: 8 * 60 * 60, report_timeout: nil, inactive_workers_timeout: nil
+        queue_init_timeout: nil, redis_ttl: 8 * 60 * 60, report_timeout: nil, inactive_workers_timeout: nil,
+        export_flaky_tests_file: nil
       )
         @build_id = build_id
         @circuit_breakers = [CircuitBreaker::Disabled]
@@ -59,6 +60,7 @@ def initialize(
         @redis_ttl = redis_ttl
         @report_timeout = report_timeout
         @inactive_workers_timeout = inactive_workers_timeout
+        @export_flaky_tests_file = export_flaky_tests_file
       end
 
       def queue_init_timeout

ruby/lib/ci/queue/redis/build_record.rb

@@ -48,10 +48,23 @@ def record_error(id, payload, stats: nil)
         end
 
         def record_success(id, stats: nil)
-          redis.pipelined do |pipeline|
+          errror_reports_deleted_count, requeued_count, _ = redis.pipelined do |pipeline|
             pipeline.hdel(key('error-reports'), id.dup.force_encoding(Encoding::BINARY))
+            pipeline.hget(key('requeues-count'), id.b)
             record_stats(stats, pipeline: pipeline)
           end
+          record_flaky(id) if errror_reports_deleted_count.to_i > 0 || requeued_count.to_i > 0
+          nil
+        end
+
+        def record_flaky(id, stats: nil)
+          redis.pipelined do |pipeline|
+            pipeline.sadd(
+              key('flaky-reports'),
+              id.b
+            )
+            pipeline.expire(key('flaky-reports'), config.redis_ttl)
+          end
           nil
         end
 
@@ -65,6 +78,10 @@ def error_reports
           redis.hgetall(key('error-reports'))
         end
 
+        def flaky_reports
+          redis.smembers(key('flaky-reports'))
+        end
+
         def fetch_stats(stat_names)
           counts = redis.pipelined do |pipeline|
             stat_names.each { |c| pipeline.hvals(key(c)) }

ruby/lib/minitest/queue/build_status_reporter.rb

@@ -17,6 +17,10 @@ def error_reports
         build.error_reports.sort_by(&:first).map { |k, v| ErrorReport.load(v) }
       end
 
+      def flaky_reports
+        build.flaky_reports
+      end
+
       def report
         puts aggregates
         errors = error_reports

ruby/lib/minitest/queue/runner.rb

@@ -213,6 +213,11 @@ def report_command
           File.write(queue_config.failure_file, failures)
         end
 
+        if queue_config.export_flaky_tests_file
+          failures = reporter.flaky_reports.to_json
+          File.write(queue_config.export_flaky_tests_file, failures)
+        end
+
         reporter.report
         exit! reporter.success? ? 0 : 1
       end
@@ -481,6 +486,15 @@ def parser
             queue_config.failure_file = file
           end
 
+          help = <<~EOS
+            Defines a file where flaky tests during the execution are written to in json format.
+            Defaults to disabled.
+          EOS
+          opts.separator ""
+          opts.on('--export-flaky-tests-file FILE', help) do |file|
+            queue_config.export_flaky_tests_file = file
+          end
+
           help = <<~EOS
             Defines after how many consecutive failures the worker will be considered unhealthy and terminate itself.
             Defaults to disabled.

ruby/test/integration/minitest_redis_test.rb

@@ -613,6 +613,44 @@ def test_redis_reporter_failure_file
       end
     end
 
+    def test_redis_reporter_flaky_tests_file
+      Dir.mktmpdir do |dir|
+        flaky_tests_file = File.join(dir, 'flaky_tests_file.json')
+
+        capture_subprocess_io do
+          system(
+            { 'BUILDKITE' => '1' },
+            @exe, 'run',
+            '--queue', @redis_url,
+            '--seed', 'foobar',
+            '--build', '1',
+            '--worker', '1',
+            '--timeout', '1',
+            '--max-requeues', '1',
+            '--requeue-tolerance', '1',
+            '-Itest',
+            'test/dummy_test.rb',
+            chdir: 'test/fixtures/',
+          )
+        end
+
+        capture_subprocess_io do
+          system(
+            @exe, 'report',
+            '--queue', @redis_url,
+            '--build', '1',
+            '--timeout', '1',
+            '--export-flaky-tests-file', flaky_tests_file,
+            chdir: 'test/fixtures/',
+          )
+        end
+
+        content = File.read(flaky_tests_file)
+        flaky_tests = JSON.parse(content)
+        assert_includes flaky_tests, "ATest#test_flaky"
+      end
+    end
+
     def test_redis_reporter
       # HACK: Simulate a timeout
       config = CI::Queue::Configuration.new(build_id: '1', worker_id: '1', timeout: '1')