From a4e542181ebe7c3ff36f892c7731555bbeea92a3 Mon Sep 17 00:00:00 2001 From: titusfortner Date: Sat, 15 Mar 2025 13:41:15 -0700 Subject: [PATCH 1/8] [build] make artifact upload handling more generic --- .github/workflows/bazel.yml | 21 +++++++++++---------- .github/workflows/pin-browsers.yml | 7 +++++-- 2 files changed, 16 insertions(+), 12 deletions(-) diff --git a/.github/workflows/bazel.yml b/.github/workflows/bazel.yml index 3f4a9f0c957a6..967f9d9d741bf 100644 --- a/.github/workflows/bazel.yml +++ b/.github/workflows/bazel.yml @@ -61,11 +61,16 @@ on: required: false type: string default: '' - artifact-name: + upload-name: description: Name of artifact to upload required: false type: string - default: 'ignore-artifacts' + default: '' + upload-path: + description: path of artifact to upload + required: false + type: string + default: '' jobs: bazel: @@ -171,14 +176,10 @@ jobs: title: "Nightly" prerelease: true files: ${{ inputs.nightly-release-files }} - - name: Save changes - if: ${{ always() && inputs.artifact-name != 'ignore-artifacts' }} - run: | - git diff > changes.patch - - name: "Upload changes" - if: ${{ always() && inputs.artifact-name != 'ignore-artifacts' }} + - name: "Upload artifact" + if: ${{ always() && inputs.upload-name != '' && inputs.upload-path != '' }} uses: actions/upload-artifact@v4 with: - name: ${{ inputs.artifact-name }} - path: changes.patch + name: ${{ inputs.upload-name }} + path: ${{ inputs.upload-path }} retention-days: 6 diff --git a/.github/workflows/pin-browsers.yml b/.github/workflows/pin-browsers.yml index 1275d3e159946..4cf7458a99947 100644 --- a/.github/workflows/pin-browsers.yml +++ b/.github/workflows/pin-browsers.yml @@ -11,8 +11,11 @@ jobs: with: name: Pin Browsers cache-key: pin-browsers - run: bazel run //scripts:pinned_browsers - artifact-name: pinned-browsers + upload-name: pinned-browsers + upload-path: changes.patch + run: | + bazel run //scripts:pinned_browsers + git diff > changes.patch pull-request: if: github.repository_owner == 'seleniumhq' From 8c7652bb9de78f7e7206e6b97d234c4705e30d68 Mon Sep 17 00:00:00 2001 From: titusfortner Date: Sun, 16 Mar 2025 11:54:05 -0700 Subject: [PATCH 2/8] [build] log rbe output to an artifact --- .github/workflows/bazel.yml | 1 + .github/workflows/ci-rbe.yml | 2 ++ scripts/github-actions/ci-build.sh | 25 +++++++++++++++++++------ 3 files changed, 22 insertions(+), 6 deletions(-) diff --git a/.github/workflows/bazel.yml b/.github/workflows/bazel.yml index 967f9d9d741bf..583ca5272ba0e 100644 --- a/.github/workflows/bazel.yml +++ b/.github/workflows/bazel.yml @@ -83,6 +83,7 @@ jobs: TWINE_PASSWORD: ${{ secrets.TWINE_PASSWORD }} TWINE_USERNAME: ${{ secrets.TWINE_USERNAME }} NODE_AUTH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + BAZEL_LOG_FILE: bazel-logs/bzl_${{ github.run_id }}.log SE_AVOID_STATS: true steps: - name: Checkout source tree diff --git a/.github/workflows/ci-rbe.yml b/.github/workflows/ci-rbe.yml index 02cf3522df8f6..125f1abf1f701 100644 --- a/.github/workflows/ci-rbe.yml +++ b/.github/workflows/ci-rbe.yml @@ -26,4 +26,6 @@ jobs: name: All RBE tests caching: false ruby-version: jruby-9.4.12.0 + upload-name: bazel-logs + upload-path: bazel-logs run: ./scripts/github-actions/ci-build.sh diff --git a/scripts/github-actions/ci-build.sh b/scripts/github-actions/ci-build.sh index 97c3a57e4fe0b..b07798a7be0d1 100755 --- a/scripts/github-actions/ci-build.sh +++ b/scripts/github-actions/ci-build.sh @@ -4,12 +4,25 @@ set -eufo pipefail # We want to see what's going on set -x -# Now run the tests. The engflow build uses pinned browsers -# so this should be fine -# shellcheck disable=SC2046 -bazel test --config=remote-ci --build_tests_only \ - --keep_going --flaky_test_attempts=2 \ - //... -- $(cat .skipped-tests | tr '\n' ' ') +run_bazel_tests() { + # shellcheck disable=SC2046 + bazel test --config=remote-ci --build_tests_only \ + --keep_going --flaky_test_attempts=2 \ + //... -- $(cat .skipped-tests | tr '\n' ' ') +} + +# Prepare log directory if BAZEL_LOG_FILE is set and run tests +if [ -n "${BAZEL_LOG_FILE:-}" ]; then + LOG_DIR=$(dirname "$BAZEL_LOG_FILE") + if mkdir -p "$LOG_DIR"; then + run_bazel_tests 2>&1 | tee "$BAZEL_LOG_FILE" + else + echo "Error: Failed to create directory for BAZEL_LOG_FILE" >&2 + exit 1 + fi +else + run_bazel_tests +fi # Build the packages we want to ship to users bazel build --config=remote-ci --build_tag_filters=release-artifact //... From 5065c3832f1582be41f5533922c8a5f10951c573 Mon Sep 17 00:00:00 2001 From: titusfortner Date: Sun, 16 Mar 2025 18:01:23 -0700 Subject: [PATCH 3/8] [build] update ruby bazel module to execute rake tasks with logs --- rake_tasks/bazel.rb | 38 +++++++++++++++++++++----------------- 1 file changed, 21 insertions(+), 17 deletions(-) diff --git a/rake_tasks/bazel.rb b/rake_tasks/bazel.rb index cdd9148764c42..f40bce4e217e6 100644 --- a/rake_tasks/bazel.rb +++ b/rake_tasks/bazel.rb @@ -7,9 +7,7 @@ require_relative 'selenium_rake/checks' module Bazel - def self.execute(kind, args, target, &block) - verbose = Rake::FileUtilsExt.verbose_flag - + def self.execute(kind, args, target, verbose: Rake::FileUtilsExt.verbose_flag, log_file: nil, &block) if target.end_with?(':run') kind = 'run' target = target[0, target.length - 4] @@ -18,28 +16,34 @@ def self.execute(kind, args, target, &block) cmd = %w[bazel] + [kind, target] + (args || []) cmd_out = '' cmd_exit_code = 0 + puts "Executing:\n#{cmd.join(' ')}" if verbose if SeleniumRake::Checks.windows? - cmd += ['2>&1'] cmd_line = cmd.join(' ') - cmd_out = `#{cmd_line}`.encode('UTF-8', 'binary', invalid: :replace, undef: :replace, replace: '') - puts cmd_out if verbose - cmd_exit_code = $CHILD_STATUS + begin + cmd_out = `#{cmd_line} 2>&1`.encode('UTF-8', 'binary', invalid: :replace, undef: :replace, replace: '') + puts cmd_out if verbose + File.write(log_file, cmd_out) if log_file + cmd_exit_code = $CHILD_STATUS.exitstatus + rescue => e + raise "Windows command execution failed: #{e.message}" + end else - Open3.popen2e(*cmd) do |stdin, stdouts, wait| - is_running = true - stdin.close - while is_running - begin - stdouts.wait_readable - line = stdouts.readpartial(512) + begin + Open3.popen2e(*cmd) do |stdin, stdouts, wait| + stdin.close + log = log_file ? File.open(log_file, 'a') : nil + while (line = stdouts.gets) cmd_out += line $stdout.print line if verbose - rescue EOFError - is_running = false + log&.write(line) + log&.flush end + log&.close + cmd_exit_code = wait.value.exitstatus end - cmd_exit_code = wait.value.exitstatus + rescue => e + raise "Command execution failed: #{e.message}" end end From 963a10d1c27059b9a69ca99f86696bc37251bf2a Mon Sep 17 00:00:00 2001 From: titusfortner Date: Sun, 16 Mar 2025 18:34:07 -0700 Subject: [PATCH 4/8] [build] implement jobs to retry rbe failures with extra debugging --- .github/workflows/bazel.yml | 16 +++++++++++++++ .github/workflows/ci-rbe.yml | 32 +++++++++++++++++++++++++++++ Rakefile | 39 ++++++++++++++++++++++++++++++++++++ 3 files changed, 87 insertions(+) diff --git a/.github/workflows/bazel.yml b/.github/workflows/bazel.yml index 583ca5272ba0e..1c94456d7da49 100644 --- a/.github/workflows/bazel.yml +++ b/.github/workflows/bazel.yml @@ -61,6 +61,16 @@ on: required: false type: string default: '' + download-name: + description: name of artifact to download + required: false + type: string + default: '' + download-path: + description: path of artifact to download + required: false + type: string + default: '' upload-name: description: Name of artifact to upload required: false @@ -161,6 +171,12 @@ jobs: - name: Setup curl for Ubuntu if: inputs.os == 'ubuntu' run: sudo apt-get update && sudo apt-get install -y libcurl4-openssl-dev + - name: "Download artifact" + if: ${{ inputs.download-name != '' && inputs.download-path != '' }} + uses: actions/download-artifact@v4 + with: + name: ${{ inputs.download-name }} + path: ${{ inputs.download-path }} - name: Run Bazel run: ${{ inputs.run }} - name: Start SSH session diff --git a/.github/workflows/ci-rbe.yml b/.github/workflows/ci-rbe.yml index 125f1abf1f701..9457ed6fad2c1 100644 --- a/.github/workflows/ci-rbe.yml +++ b/.github/workflows/ci-rbe.yml @@ -29,3 +29,35 @@ jobs: upload-name: bazel-logs upload-path: bazel-logs run: ./scripts/github-actions/ci-build.sh + retry-remote: + name: Retry Failures on RBE + needs: test + if: always() && needs.test.result == 'failure' && github.repository_owner == 'seleniumhq' && startsWith(github.head_ref, 'renovate/') != true + uses: ./.github/workflows/bazel.yml + with: + name: RBE Retries + caching: false + ruby-version: jruby-9.4.12.0 + download-name: bazel-logs + download-path: bazel-logs + upload-name: retry-remote-logs + upload-path: bazel-logs + run: | + ./go retry_failed_tests "${BAZEL_LOG_FILE}" true + retry-local: + name: Retry Failures on GHA + needs: test + if: always() && needs.test.result == 'failure' && github.repository_owner == 'seleniumhq' && startsWith(github.head_ref, 'renovate/') != true + uses: ./.github/workflows/bazel.yml + with: + name: GHA Retries + caching: false + ruby-version: jruby-9.4.12.0 + browser: needs-xvfb + java-version: 17 + download-name: bazel-logs + download-path: bazel-logs + upload-name: retry-local-logs + upload-path: bazel-logs + run: | + ./go retry_failed_tests "${BAZEL_LOG_FILE}" false diff --git a/Rakefile b/Rakefile index a58182c24d465..d5a68dfd22627 100644 --- a/Rakefile +++ b/Rakefile @@ -274,6 +274,45 @@ end task test_py: [:py_prep_for_install_release, 'py:marionette_test'] task build: %i[all firefox remote selenium tests] +desc 'Retry failed tests from a log file' +task :retry_failed_tests, [:log_file, :rbe] do |_task, arguments| + log_file = arguments[:log_file] + + raise 'Error: Please provide a bazel log file containing test results.' if log_file.nil? + raise "Error: Log file '#{log_file}' does not exist." unless File.exist?(log_file) + + rbe = arguments[:rbe] + failing_tests = [] + + File.readlines(log_file).reverse_each do |line| + if line.match?(%r{//.+:.*FAILED}) + failing_tests << line.strip.split[0] + elsif !failing_tests.empty? && line.match?(%r{//.+:.*PASSED}) + break + end + end + + puts "Found #{failing_tests.size} failing tests; Retrying" + + retry_args = arguments.extras + %w[--test_output=streamed --test_env DEBUG=true] + retry_args << (rbe == 'true' ? '--config=remote' : '--pin_browsers') + + retry_failures = false + failing_tests.each do |failed_target| + target_name = failed_target.split('/').last.tr(':', '_') + target_name += rbe ? '_rbe' : '_gha' + retry_logs = log_file.sub('.log', "_#{target_name}_retry.log") + begin + Bazel.execute('test', retry_args, failed_target, verbose: true, log_file: retry_logs) + rescue StandardError => e + retry_failures = true + puts "Failed retry of #{failed_target}: #{e.message}" + end + end + + raise 'Some tests failed during retry' if retry_failures +end + desc 'Clean build artifacts.' task :clean do rm_rf 'build/' From ee541c2a37aab40640664ed93473f13f1e257ddf Mon Sep 17 00:00:00 2001 From: titusfortner Date: Sun, 16 Mar 2025 20:14:14 -0700 Subject: [PATCH 5/8] [build] only enable command tracing for rbe run in Debug mode --- scripts/github-actions/ci-build.sh | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/scripts/github-actions/ci-build.sh b/scripts/github-actions/ci-build.sh index b07798a7be0d1..14bace1266375 100755 --- a/scripts/github-actions/ci-build.sh +++ b/scripts/github-actions/ci-build.sh @@ -1,8 +1,11 @@ #!/usr/bin/env bash set -eufo pipefail -# We want to see what's going on -set -x + +# Only enable command tracing if DEBUG is set +if [ -n "${DEBUG:-}" ]; then + set -x +fi run_bazel_tests() { # shellcheck disable=SC2046 From 207ec2379b240ce853c184919feb7a649e670fc2 Mon Sep 17 00:00:00 2001 From: titusfortner Date: Sat, 15 Mar 2025 19:31:05 -0700 Subject: [PATCH 6/8] [dotnet] toggle debug logs for tests based on environment variable DEBUG --- dotnet/test/common/Environment/EnvironmentManager.cs | 8 +++++++- dotnet/test/common/Environment/TestEnvironment.cs | 3 +++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/dotnet/test/common/Environment/EnvironmentManager.cs b/dotnet/test/common/Environment/EnvironmentManager.cs index 9c0b53b90c56c..7afe0ade99f5d 100644 --- a/dotnet/test/common/Environment/EnvironmentManager.cs +++ b/dotnet/test/common/Environment/EnvironmentManager.cs @@ -25,6 +25,7 @@ using System.IO; using System.Linq; using System.Runtime.InteropServices; +using OpenQA.Selenium.Internal.Logging; namespace OpenQA.Selenium.Environment { @@ -60,8 +61,13 @@ private EnvironmentManager() string activeDriverConfig = System.Environment.GetEnvironmentVariable("ACTIVE_DRIVER_CONFIG") ?? TestContext.Parameters.Get("ActiveDriverConfig", env.ActiveDriverConfig); string driverServiceLocation = System.Environment.GetEnvironmentVariable("DRIVER_SERVICE_LOCATION") ?? TestContext.Parameters.Get("DriverServiceLocation", env.DriverServiceLocation); - string browserLocation = System.Environment.GetEnvironmentVariable("BROWSER_LOCATION") ?? TestContext.Parameters.Get("BrowserLocation", string.Empty); + string enableDebugging = System.Environment.GetEnvironmentVariable("DEBUG") ?? TestContext.Parameters.Get("Debug", env.Debug); + + if (!string.IsNullOrEmpty(enableDebugging)) + { + Log.SetLevel(LogEventLevel.Debug); + } string activeWebsiteConfig = TestContext.Parameters.Get("ActiveWebsiteConfig", env.ActiveWebsiteConfig); DriverConfig driverConfig = env.DriverConfigs[activeDriverConfig]; diff --git a/dotnet/test/common/Environment/TestEnvironment.cs b/dotnet/test/common/Environment/TestEnvironment.cs index 40bffe0c739b3..badb16dbde92e 100644 --- a/dotnet/test/common/Environment/TestEnvironment.cs +++ b/dotnet/test/common/Environment/TestEnvironment.cs @@ -48,5 +48,8 @@ class TestEnvironment [JsonProperty] public TestWebServerConfig TestWebServerConfig { get; set; } + + [JsonProperty] + public string Debug { get; set; } } } From e782cb645aafb85459ed9f6b59c4efcc89c7f970 Mon Sep 17 00:00:00 2001 From: titusfortner Date: Sat, 15 Mar 2025 19:21:51 -0700 Subject: [PATCH 7/8] [py] enable selenium logging in tests if environment variable DEBUG is set --- py/conftest.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/py/conftest.py b/py/conftest.py index a16c83b3e0226..e812bfcda5d95 100644 --- a/py/conftest.py +++ b/py/conftest.py @@ -14,7 +14,7 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. - +import logging import os import platform import socket @@ -140,6 +140,9 @@ def fin(): driver_path = request.config.option.executable options = None + if os.environ.get("DEBUG"): + logging.getLogger("selenium").setLevel(logging.DEBUG) + global driver_instance if driver_instance is None: if driver_class == "Firefox": From d70a841505b1eb9eb6ffd9b763be0b22a7fe0581 Mon Sep 17 00:00:00 2001 From: titusfortner Date: Sat, 15 Mar 2025 18:30:30 -0700 Subject: [PATCH 8/8] [java] support DEBUG environment variable for turning on logging of tests --- .../openqa/selenium/testing/JupiterTestBase.java | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/java/test/org/openqa/selenium/testing/JupiterTestBase.java b/java/test/org/openqa/selenium/testing/JupiterTestBase.java index a540bafb7c849..e8b4076c06caf 100644 --- a/java/test/org/openqa/selenium/testing/JupiterTestBase.java +++ b/java/test/org/openqa/selenium/testing/JupiterTestBase.java @@ -22,7 +22,9 @@ import java.net.MalformedURLException; import java.net.URL; import java.time.Duration; +import java.util.Arrays; import java.util.Optional; +import java.util.logging.Level; import java.util.logging.Logger; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeAll; @@ -43,6 +45,20 @@ public abstract class JupiterTestBase { @RegisterExtension protected static SeleniumExtension seleniumExtension = new SeleniumExtension(); + static { + if ("true".equalsIgnoreCase(System.getenv("DEBUG"))) { + Logger rootLogger = Logger.getLogger(""); + rootLogger.setLevel(Level.FINE); + Arrays.stream(rootLogger.getHandlers()) + .forEach( + handler -> { + handler.setLevel(Level.FINE); + }); + + LOG.fine("Global debug logging enabled via DEBUG environment variable"); + } + } + protected TestEnvironment environment; protected AppServer appServer; protected Pages pages;