Retry failed RBE tests and get verbose logs

.github/workflows/bazel.yml

@@ -61,11 +61,26 @@ on:
         required: false
         type: string
         default: ''
-      artifact-name:
+      download-name:
+        description: name of artifact to download
+        required: false
+        type: string
+        default: ''
+      download-path:
+        description: path of artifact to download
+        required: false
+        type: string
+        default: ''
+      upload-name:
         description: Name of artifact to upload
         required: false
         type: string
-        default: 'ignore-artifacts'
+        default: ''
+      upload-path:
+        description: path of artifact to upload
+        required: false
+        type: string
+        default: ''
 
 jobs:
   bazel:
@@ -78,6 +93,7 @@ jobs:
       TWINE_PASSWORD: ${{ secrets.TWINE_PASSWORD }}
       TWINE_USERNAME: ${{ secrets.TWINE_USERNAME }}
       NODE_AUTH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      BAZEL_LOG_FILE: bazel-logs/bzl_${{ github.run_id }}.log
       SE_AVOID_STATS: true
     steps:
       - name: Checkout source tree
@@ -155,6 +171,12 @@ jobs:
       - name: Setup curl for Ubuntu
         if: inputs.os == 'ubuntu'
         run: sudo apt-get update && sudo apt-get install -y libcurl4-openssl-dev
+      - name: "Download artifact"
+        if: ${{ inputs.download-name != '' && inputs.download-path != '' }}
+        uses: actions/download-artifact@v4
+        with:
+          name: ${{ inputs.download-name }}
+          path: ${{ inputs.download-path }}
       - name: Run Bazel
         run: ${{ inputs.run }}
       - name: Start SSH session
@@ -171,14 +193,10 @@ jobs:
           title: "Nightly"
           prerelease: true
           files: ${{ inputs.nightly-release-files }}
-      - name: Save changes
-        if: ${{ always() && inputs.artifact-name != 'ignore-artifacts' }}
-        run: |
-          git diff > changes.patch
-      - name: "Upload changes"
-        if: ${{ always() && inputs.artifact-name != 'ignore-artifacts' }}
+      - name: "Upload artifact"
+        if: ${{ always() && inputs.upload-name != '' && inputs.upload-path != '' }}
         uses: actions/upload-artifact@v4
         with:
-          name: ${{ inputs.artifact-name }}
-          path: changes.patch
+          name: ${{ inputs.upload-name }}
+          path: ${{ inputs.upload-path }}
           retention-days: 6

.github/workflows/ci-rbe.yml

@@ -26,4 +26,38 @@ jobs:
       name: All RBE tests
       caching: false
       ruby-version: jruby-9.4.12.0
+      upload-name: bazel-logs
+      upload-path: bazel-logs
       run: ./scripts/github-actions/ci-build.sh
+  retry-remote:
+    name: Retry Failures on RBE
+    needs: test
+    if: always() && needs.test.result == 'failure' && github.repository_owner == 'seleniumhq' && startsWith(github.head_ref, 'renovate/') != true
+    uses: ./.github/workflows/bazel.yml
+    with:
+      name: RBE Retries
+      caching: false
+      ruby-version: jruby-9.4.12.0
+      download-name: bazel-logs
+      download-path: bazel-logs
+      upload-name: retry-remote-logs
+      upload-path: bazel-logs
+      run: |
+        ./go retry_failed_tests "${BAZEL_LOG_FILE}" true
+  retry-local:
+    name: Retry Failures on GHA
+    needs: test
+    if: always() && needs.test.result == 'failure' && github.repository_owner == 'seleniumhq' && startsWith(github.head_ref, 'renovate/') != true
+    uses: ./.github/workflows/bazel.yml
+    with:
+      name: GHA Retries
+      caching: false
+      ruby-version: jruby-9.4.12.0
+      browser: needs-xvfb
+      java-version: 17
+      download-name: bazel-logs
+      download-path: bazel-logs
+      upload-name: retry-local-logs
+      upload-path: bazel-logs
+      run: |
+        ./go retry_failed_tests "${BAZEL_LOG_FILE}" false

.github/workflows/pin-browsers.yml

@@ -11,8 +11,11 @@ jobs:
     with:
       name: Pin Browsers
       cache-key: pin-browsers
-      run: bazel run //scripts:pinned_browsers
-      artifact-name: pinned-browsers
+      upload-name: pinned-browsers
+      upload-path: changes.patch
+      run: |
+        bazel run //scripts:pinned_browsers
+        git diff > changes.patch
 
   pull-request:
     if: github.repository_owner == 'seleniumhq'

Rakefile

@@ -274,6 +274,45 @@ end
 task test_py: [:py_prep_for_install_release, 'py:marionette_test']
 task build: %i[all firefox remote selenium tests]
 
+desc 'Retry failed tests from a log file'
+task :retry_failed_tests, [:log_file, :rbe] do |_task, arguments|
+  log_file = arguments[:log_file]
+
+  raise 'Error: Please provide a bazel log file containing test results.' if log_file.nil?
+  raise "Error: Log file '#{log_file}' does not exist." unless File.exist?(log_file)
+
+  rbe = arguments[:rbe]
+  failing_tests = []
+
+  File.readlines(log_file).reverse_each do |line|
+    if line.match?(%r{//.+:.*FAILED})
+      failing_tests << line.strip.split[0]
+    elsif !failing_tests.empty? && line.match?(%r{//.+:.*PASSED})
+      break
+    end
+  end
+
+  puts "Found #{failing_tests.size} failing tests; Retrying"
+
+  retry_args = arguments.extras + %w[--test_output=streamed --test_env DEBUG=true]
+  retry_args << (rbe == 'true' ? '--config=remote' : '--pin_browsers')
+
+  retry_failures = false
+  failing_tests.each do |failed_target|
+    target_name = failed_target.split('/').last.tr(':', '_')
+    target_name += rbe ? '_rbe' : '_gha'
+    retry_logs = log_file.sub('.log', "_#{target_name}_retry.log")
+    begin
+      Bazel.execute('test', retry_args, failed_target, verbose: true, log_file: retry_logs)
+    rescue StandardError => e
+      retry_failures = true
+      puts "Failed retry of #{failed_target}: #{e.message}"
+    end
+  end
+
+  raise 'Some tests failed during retry' if retry_failures
+end
+
 desc 'Clean build artifacts.'
 task :clean do
   rm_rf 'build/'

dotnet/test/common/Environment/EnvironmentManager.cs

@@ -25,6 +25,7 @@
 using System.IO;
 using System.Linq;
 using System.Runtime.InteropServices;
+using OpenQA.Selenium.Internal.Logging;
 
 namespace OpenQA.Selenium.Environment
 {
@@ -60,8 +61,13 @@ private EnvironmentManager()
 
             string activeDriverConfig = System.Environment.GetEnvironmentVariable("ACTIVE_DRIVER_CONFIG") ?? TestContext.Parameters.Get("ActiveDriverConfig", env.ActiveDriverConfig);
             string driverServiceLocation = System.Environment.GetEnvironmentVariable("DRIVER_SERVICE_LOCATION") ?? TestContext.Parameters.Get("DriverServiceLocation", env.DriverServiceLocation);
-
             string browserLocation = System.Environment.GetEnvironmentVariable("BROWSER_LOCATION") ?? TestContext.Parameters.Get("BrowserLocation", string.Empty);
+            string enableDebugging = System.Environment.GetEnvironmentVariable("DEBUG") ?? TestContext.Parameters.Get("Debug", env.Debug);
+
+            if (!string.IsNullOrEmpty(enableDebugging))
+            {
+                Log.SetLevel(LogEventLevel.Debug);
+            }
 
             string activeWebsiteConfig = TestContext.Parameters.Get("ActiveWebsiteConfig", env.ActiveWebsiteConfig);
             DriverConfig driverConfig = env.DriverConfigs[activeDriverConfig];

dotnet/test/common/Environment/TestEnvironment.cs

@@ -48,5 +48,8 @@ class TestEnvironment
 
         [JsonProperty]
         public TestWebServerConfig TestWebServerConfig { get; set; }
+
+        [JsonProperty]
+        public string Debug { get; set; }
     }
 }

java/test/org/openqa/selenium/testing/JupiterTestBase.java

@@ -22,7 +22,9 @@
 import java.net.MalformedURLException;
 import java.net.URL;
 import java.time.Duration;
+import java.util.Arrays;
 import java.util.Optional;
+import java.util.logging.Level;
 import java.util.logging.Logger;
 import org.junit.jupiter.api.AfterEach;
 import org.junit.jupiter.api.BeforeAll;
@@ -43,6 +45,20 @@ public abstract class JupiterTestBase {
 
   @RegisterExtension protected static SeleniumExtension seleniumExtension = new SeleniumExtension();
 
+  static {
+    if ("true".equalsIgnoreCase(System.getenv("DEBUG"))) {
+      Logger rootLogger = Logger.getLogger("");
+      rootLogger.setLevel(Level.FINE);
+      Arrays.stream(rootLogger.getHandlers())
+          .forEach(
+              handler -> {
+                handler.setLevel(Level.FINE);
+              });
+
+      LOG.fine("Global debug logging enabled via DEBUG environment variable");
+    }
+  }
+
   protected TestEnvironment environment;
   protected AppServer appServer;
   protected Pages pages;

py/conftest.py

@@ -14,7 +14,7 @@
 # KIND, either express or implied.  See the License for the
 # specific language governing permissions and limitations
 # under the License.
-
+import logging
 import os
 import platform
 import socket
@@ -140,6 +140,9 @@ def fin():
     driver_path = request.config.option.executable
     options = None
 
+    if os.environ.get("DEBUG"):
+        logging.getLogger("selenium").setLevel(logging.DEBUG)
+
     global driver_instance
     if driver_instance is None:
         if driver_class == "Firefox":

rake_tasks/bazel.rb

@@ -7,9 +7,7 @@
 require_relative 'selenium_rake/checks'
 
 module Bazel
-  def self.execute(kind, args, target, &block)
-    verbose = Rake::FileUtilsExt.verbose_flag
-
+  def self.execute(kind, args, target, verbose: Rake::FileUtilsExt.verbose_flag, log_file: nil, &block)
     if target.end_with?(':run')
       kind = 'run'
       target = target[0, target.length - 4]
@@ -18,28 +16,34 @@ def self.execute(kind, args, target, &block)
     cmd = %w[bazel] + [kind, target] + (args || [])
     cmd_out = ''
     cmd_exit_code = 0
+    puts "Executing:\n#{cmd.join(' ')}" if verbose
 
     if SeleniumRake::Checks.windows?
-      cmd += ['2>&1']
       cmd_line = cmd.join(' ')
-      cmd_out = `#{cmd_line}`.encode('UTF-8', 'binary', invalid: :replace, undef: :replace, replace: '')
-      puts cmd_out if verbose
-      cmd_exit_code = $CHILD_STATUS
+      begin
+        cmd_out = `#{cmd_line} 2>&1`.encode('UTF-8', 'binary', invalid: :replace, undef: :replace, replace: '')
+        puts cmd_out if verbose
+        File.write(log_file, cmd_out) if log_file
+        cmd_exit_code = $CHILD_STATUS.exitstatus
+      rescue => e
+        raise "Windows command execution failed: #{e.message}"
+      end
     else
-      Open3.popen2e(*cmd) do |stdin, stdouts, wait|
-        is_running = true
-        stdin.close
-        while is_running
-          begin
-            stdouts.wait_readable
-            line = stdouts.readpartial(512)
+      begin
+        Open3.popen2e(*cmd) do |stdin, stdouts, wait|
+          stdin.close
+          log = log_file ? File.open(log_file, 'a') : nil
+          while (line = stdouts.gets)
             cmd_out += line
             $stdout.print line if verbose
-          rescue EOFError
-            is_running = false
+            log&.write(line)
+            log&.flush
           end
+          log&.close
+          cmd_exit_code = wait.value.exitstatus
         end
-        cmd_exit_code = wait.value.exitstatus
+      rescue => e
+        raise "Command execution failed: #{e.message}"
       end
     end
 

scripts/github-actions/ci-build.sh

@@ -1,15 +1,31 @@
 #!/usr/bin/env bash
 
 set -eufo pipefail
-# We want to see what's going on
-set -x
-
-# Now run the tests. The engflow build uses pinned browsers
-# so this should be fine
-# shellcheck disable=SC2046
-bazel test --config=remote-ci --build_tests_only \
-  --keep_going --flaky_test_attempts=2 \
-  //... -- $(cat .skipped-tests | tr '\n' ' ')
+
+# Only enable command tracing if DEBUG is set
+if [ -n "${DEBUG:-}" ]; then
+  set -x
+fi
+
+run_bazel_tests() {
+  # shellcheck disable=SC2046
+  bazel test --config=remote-ci --build_tests_only \
+    --keep_going --flaky_test_attempts=2 \
+    //... -- $(cat .skipped-tests | tr '\n' ' ')
+}
+
+# Prepare log directory if BAZEL_LOG_FILE is set and run tests
+if [ -n "${BAZEL_LOG_FILE:-}" ]; then
+  LOG_DIR=$(dirname "$BAZEL_LOG_FILE")
+  if mkdir -p "$LOG_DIR"; then
+    run_bazel_tests 2>&1 | tee "$BAZEL_LOG_FILE"
+  else
+    echo "Error: Failed to create directory for BAZEL_LOG_FILE" >&2
+    exit 1
+  fi
+else
+  run_bazel_tests
+fi
 
 # Build the packages we want to ship to users
 bazel build --config=remote-ci --build_tag_filters=release-artifact //...