Fix leakbot command instructio for CI="true"

Resolves: Shopify/shopify#421914
Shopify · May 2, 2023 · 32ab5e4 · 32ab5e4
1 parent 10d895f
commit 32ab5e4
Show file tree

Hide file tree

Showing 2 changed files with 101 additions and 3 deletions.
diff --git a/ruby/lib/minitest/queue/runner.rb b/ruby/lib/minitest/queue/runner.rb
@@ -170,7 +170,7 @@ def bisect_command
           exit! 1
         else
           step(green('The following command should reproduce the leak on your machine:'), collapsed: false)
-          command = %w(bundle exec minitest-queue --queue - run)
+          command = reproduce_command
           command << "-I#{load_paths}" if load_paths
           command += argv
 
@@ -582,6 +582,14 @@ def retry?
         ENV["BUILDKITE_RETRY_COUNT"].to_i > 0 ||
           ENV["SEMAPHORE_PIPELINE_RERUN"] == "true"
       end
+
+      def reproduce_command
+        if ENV['CI'] != 'true' || queue.is_a?(CI::Queue::File)
+          %w(bundle exec minitest-queue --queue - run --build local --worker local)
+        else
+          %w(bundle exec minitest-queue --queue - run)
+        end
+      end
     end
   end
 end
diff --git a/ruby/test/integration/minitest_bisect_test.rb b/ruby/test/integration/minitest_bisect_test.rb
@@ -15,6 +15,96 @@ def test_bisect
         run_bisect('log/leaky_test_order.log', 'LeakyTest#test_sensible_to_leak')
       end
 
+      assert_empty err
+      expected_output = strip_heredoc <<-EOS
+        --- Testing the failing test in isolation
+          LeakyTest#test_sensible_to_leak                                 PASS
+        --- Run #1, 45 suspects left
+          LeakyTest#test_useless_0                                        PASS
+          LeakyTest#test_useless_1                                        PASS
+          LeakyTest#test_useless_2                                        PASS
+          LeakyTest#test_useless_3                                        PASS
+          LeakyTest#test_useless_4                                        PASS
+          LeakyTest#test_useless_5                                        PASS
+          LeakyTest#test_useless_6                                        PASS
+          LeakyTest#test_useless_7                                        PASS
+          LeakyTest#test_useless_8                                        PASS
+          LeakyTest#test_useless_9                                        PASS
+          LeakyTest#test_introduce_leak                                   PASS
+          LeakyTest#test_useless_10                                       PASS
+          LeakyTest#test_useless_11                                       PASS
+          LeakyTest#test_useless_12                                       PASS
+          LeakyTest#test_useless_13                                       PASS
+          LeakyTest#test_useless_14                                       PASS
+          LeakyTest#test_useless_15                                       PASS
+          LeakyTest#test_useless_16                                       PASS
+          LeakyTest#test_useless_17                                       PASS
+          LeakyTest#test_useless_18                                       PASS
+          LeakyTest#test_useless_19                                       PASS
+          LeakyTest#test_useless_20                                       PASS
+          LeakyTest#test_useless_21                                       PASS
+          LeakyTest#test_sensible_to_leak                                 FAIL
+
+        --- Run #2, 23 suspects left
+          LeakyTest#test_useless_0                                        PASS
+          LeakyTest#test_useless_1                                        PASS
+          LeakyTest#test_useless_2                                        PASS
+          LeakyTest#test_useless_3                                        PASS
+          LeakyTest#test_useless_4                                        PASS
+          LeakyTest#test_useless_5                                        PASS
+          LeakyTest#test_useless_6                                        PASS
+          LeakyTest#test_useless_7                                        PASS
+          LeakyTest#test_useless_8                                        PASS
+          LeakyTest#test_useless_9                                        PASS
+          LeakyTest#test_introduce_leak                                   PASS
+          LeakyTest#test_useless_10                                       PASS
+          LeakyTest#test_sensible_to_leak                                 FAIL
+
+        --- Run #3, 12 suspects left
+          LeakyTest#test_useless_0                                        PASS
+          LeakyTest#test_useless_1                                        PASS
+          LeakyTest#test_useless_2                                        PASS
+          LeakyTest#test_useless_3                                        PASS
+          LeakyTest#test_useless_4                                        PASS
+          LeakyTest#test_useless_5                                        PASS
+          LeakyTest#test_sensible_to_leak                                 PASS
+
+        --- Run #4, 6 suspects left
+          LeakyTest#test_useless_6                                        PASS
+          LeakyTest#test_useless_7                                        PASS
+          LeakyTest#test_useless_8                                        PASS
+          LeakyTest#test_sensible_to_leak                                 PASS
+
+        --- Run #5, 3 suspects left
+          LeakyTest#test_useless_9                                        PASS
+          LeakyTest#test_introduce_leak                                   PASS
+          LeakyTest#test_sensible_to_leak                                 FAIL
+
+        --- Run #6, 2 suspects left
+          LeakyTest#test_useless_9                                        PASS
+          LeakyTest#test_sensible_to_leak                                 PASS
+
+        --- Final validation
+          LeakyTest#test_introduce_leak                                   PASS
+          LeakyTest#test_sensible_to_leak                                 FAIL
+        +++ The following command should reproduce the leak on your machine:
+
+        cat <<EOF |
+        LeakyTest#test_introduce_leak
+        LeakyTest#test_sensible_to_leak
+        EOF
+        bundle exec minitest-queue --queue - run --build local --worker local -Itest test/leaky_test.rb
+
+      EOS
+
+      assert_equal expected_output, normalize(out)
+    end
+
+    def test_bisect_in_ci
+      out, err = capture_subprocess_io do
+        run_bisect('log/leaky_test_order.log', 'LeakyTest#test_sensible_to_leak', ci: true)
+      end
+
       assert_empty err
       expected_output = strip_heredoc <<-EOS
         --- Testing the failing test in isolation
@@ -200,10 +290,10 @@ def normalize(output)
       rewrite_paths(freeze_seed(freeze_timing(decolorize_output(output))))
     end
 
-    def run_bisect(test_order_file, failing_test)
+    def run_bisect(test_order_file, failing_test, ci: false)
       exe = File.expand_path('../../../exe/minitest-queue', __FILE__)
       system(
-        { 'BUILDKITE' => '1' },
+        { 'BUILDKITE' => '1', 'CI' => ci.to_s },
         exe, 'bisect',
         '--queue', test_order_file,
         '--failing-test', failing_test,