From b772d75f7c0c8ba5f3d14a5938d2ded1fcfd8e40 Mon Sep 17 00:00:00 2001 From: "trop[bot]" <37223003+trop[bot]@users.noreply.github.com> Date: Thu, 11 Jan 2024 08:20:41 +0000 Subject: [PATCH] build: add infra for reclient support * chore: add patch to always set macos platform for x-build * build: add infra for reclient support * build: override reclient version * build: use RBE in CI * chore: hardcode reclient fix version * build: lower process count on macOS * build: use large macOS instance for testing-arm64 * Revert "build: use large macOS instance for testing-arm64" This reverts commit 6844adfd00a5230e68234112dfd84caa50d3f621. * build: login in via helper not writing file * chore: update patches * build: use recelint from DEPS * build: fix windows reproxy cfg * build: use reclient in appveyor * build: update WOA job too * build: force another build * build: do not checkout reclient Co-authored-by: Samuel Attard --- .circleci/config/base.yml | 110 +++++-------- DEPS | 7 + appveyor-bake.yml | 1 - appveyor-woa.yml | 51 +++---- appveyor.yml | 51 +++---- patches/chromium/.patches | 1 + ..._reclient_cfg_generator_after_chrome.patch | 30 ++++ patches/config.json | 4 +- patches/reclient-configs/.patches | 3 + ...ide_clang_wrapper_as_toolchain_input.patch | 19 +++ .../fix_add_python_remote_wrapper.patch | 144 ++++++++++++++++++ .../fix_disable_compression_threshold.patch | 17 +++ script/prepare-appveyor.js | 1 - script/start-goma.ps1 | 10 -- 14 files changed, 301 insertions(+), 148 deletions(-) create mode 100644 patches/chromium/build_run_reclient_cfg_generator_after_chrome.patch create mode 100644 patches/reclient-configs/.patches create mode 100644 patches/reclient-configs/build_on_windows_provide_clang_wrapper_as_toolchain_input.patch create mode 100644 patches/reclient-configs/fix_add_python_remote_wrapper.patch create mode 100644 patches/reclient-configs/fix_disable_compression_threshold.patch delete mode 100755 script/start-goma.ps1 diff --git a/.circleci/config/base.yml b/.circleci/config/base.yml index 35529cd53f60e..c29c61381ebfd 100644 --- a/.circleci/config/base.yml +++ b/.circleci/config/base.yml @@ -337,46 +337,26 @@ step-setup-env-for-build: &step-setup-env-for-build # To find `gn` executable. echo 'export CHROMIUM_BUILDTOOLS_PATH="'"$PWD"'/src/buildtools"' >> $BASH_ENV -step-setup-goma-for-build: &step-setup-goma-for-build +step-setup-rbe-for-build: &step-setup-rbe-for-build run: - name: Setup Goma + name: Setup RBE command: | echo 'export NUMBER_OF_NINJA_PROCESSES=300' >> $BASH_ENV if [ "`uname`" == "Darwin" ]; then + echo 'export NUMBER_OF_NINJA_PROCESSES=200' >> $BASH_ENV echo 'ulimit -n 10000' >> $BASH_ENV echo 'sudo launchctl limit maxfiles 65536 200000' >> $BASH_ENV fi - if [ ! -z "$RAW_GOMA_AUTH" ]; then - echo $RAW_GOMA_AUTH > ~/.goma_oauth2_config - fi git clone https://github.com/electron/build-tools.git cd build-tools - npm install + npx yarn --ignore-engines mkdir third_party - node -e "require('./src/utils/goma.js').downloadAndPrepare({ gomaOneForAll: true })" - export GOMA_FALLBACK_ON_AUTH_FAILURE=true - third_party/goma/goma_ctl.py ensure_start - if [ ! -z "$RAW_GOMA_AUTH" ] && [ "`third_party/goma/goma_auth.py info`" != "Login as Fermi Planck" ]; then - echo "WARNING!!!!!! Goma authentication is incorrect; please update Goma auth token." - exit 1 - fi - echo 'export GN_GOMA_FILE='`node -e "console.log(require('./src/utils/goma.js').gnFilePath)"` >> $BASH_ENV - echo 'export GOMA_DIR='`node -e "console.log(require('./src/utils/goma.js').dir)"` >> $BASH_ENV - echo 'export GOMA_FALLBACK_ON_AUTH_FAILURE=true' >> $BASH_ENV - cd .. - touch "${TMPDIR:=/tmp}"/.goma-ready - background: true - -step-wait-for-goma: &step-wait-for-goma - run: - name: Wait for Goma - command: | - until [ -f "${TMPDIR:=/tmp}"/.goma-ready ] - do - sleep 5 - done - echo "Goma ready" - no_output_timeout: 5m + # Pull down credential helper and print status + node -e "require('./src/utils/reclient.js').downloadAndPrepare({})" + HELPER=$(node -p "require('./src/utils/reclient.js').helperPath") + $HELPER login + echo 'export RBE_service='`node -e "console.log(require('./src/utils/reclient.js').serviceAddress)"` >> $BASH_ENV + echo 'export RBE_experimental_credentials_helper='`node -e "console.log(require('./src/utils/reclient.js').helperPath)"` >> $BASH_ENV step-restore-brew-cache: &step-restore-brew-cache restore_cache: @@ -550,6 +530,13 @@ step-fix-sync: &step-fix-sync sed -i '' "s/Updating depot_tools... //g" gn_ensure_file cipd ensure --root src/buildtools/mac -ensure-file gn_ensure_file + # Fix reclient (wrong binary) + echo 'infra/rbe/client/${platform}' `gclient getdep --deps-file=src/DEPS -r 'src/buildtools/reclient:infra/rbe/client/${platform}'` > gn_ensure_file + # Remove extra output from calling gclient getdep which always calls update_depot_tools + sed -i '' "s/Updating depot_tools... //g" gn_ensure_file + cipd ensure --root src/buildtools/reclient -ensure-file gn_ensure_file + python3 src/buildtools/reclient_cfgs/configure_reclient_cfgs.py --rbe_instance "projects/rbe-chrome-untrusted/instances/default_instance" --reproxy_cfg_template reproxy.cfg.template --rewrapper_cfg_project "" --skip_remoteexec_cfg_fetch + # Fix dsymutil (wrong binary) if [ "$TARGET_ARCH" == "arm64" ]; then export DSYM_SHA_FILE=src/tools/clang/dsymutil/bin/dsymutil.arm64.sha1 @@ -602,7 +589,7 @@ step-gn-gen-default: &step-gn-gen-default name: Default GN gen command: | cd src - gn gen out/Default --args="import(\"$GN_CONFIG\") import(\"$GN_GOMA_FILE\") $GN_EXTRA_ARGS $GN_BUILDFLAG_ARGS" + gn gen out/Default --args="import(\"$GN_CONFIG\") use_remoteexec=true $GN_EXTRA_ARGS $GN_BUILDFLAG_ARGS" step-gn-check: &step-gn-check run: @@ -638,16 +625,16 @@ step-electron-chromedriver-build: &step-electron-chromedriver-build command: | cd src if [ "`uname`" != "Darwin" ] && ([ "$TARGET_ARCH" == "arm" ] || [ "$TARGET_ARCH" == "arm64" ]); then - gn gen out/chromedriver --args="import(\"$GN_CONFIG\") import(\"$GN_GOMA_FILE\") is_component_ffmpeg=false proprietary_codecs=false $GN_EXTRA_ARGS $GN_BUILDFLAG_ARGS" + gn gen out/chromedriver --args="import(\"$GN_CONFIG\") use_remoteexec=true is_component_ffmpeg=false proprietary_codecs=false $GN_EXTRA_ARGS $GN_BUILDFLAG_ARGS" export CHROMEDRIVER_DIR="out/chromedriver" else export CHROMEDRIVER_DIR="out/Default" fi - ninja -C $CHROMEDRIVER_DIR electron:electron_chromedriver -j $NUMBER_OF_NINJA_PROCESSES + autoninja -C $CHROMEDRIVER_DIR electron:electron_chromedriver -j $NUMBER_OF_NINJA_PROCESSES if [ "`uname`" == "Linux" ]; then electron/script/strip-binaries.py --target-cpu="$TARGET_ARCH" --file $PWD/$CHROMEDRIVER_DIR/chromedriver fi - ninja -C $CHROMEDRIVER_DIR electron:electron_chromedriver_zip + autoninja -C $CHROMEDRIVER_DIR electron:electron_chromedriver_zip if [ "`uname`" != "Darwin" ] && ([ "$TARGET_ARCH" == "arm" ] || [ "$TARGET_ARCH" == "arm64" ]); then cp out/chromedriver/chromedriver.zip out/Default fi @@ -657,7 +644,7 @@ step-nodejs-headers-build: &step-nodejs-headers-build name: Build Node.js headers command: | cd src - ninja -C out/Default electron:node_headers + autoninja -C out/Default electron:node_headers step-electron-publish: &step-electron-publish run: @@ -711,14 +698,14 @@ step-ffmpeg-gn-gen: &step-ffmpeg-gn-gen name: ffmpeg GN gen command: | cd src - gn gen out/ffmpeg --args="import(\"//electron/build/args/ffmpeg.gn\") import(\"$GN_GOMA_FILE\") $GN_EXTRA_ARGS" + gn gen out/ffmpeg --args="import(\"//electron/build/args/ffmpeg.gn\") use_remoteexec=true $GN_EXTRA_ARGS" step-ffmpeg-build: &step-ffmpeg-build run: name: Non proprietary ffmpeg build command: | cd src - ninja -C out/ffmpeg electron:electron_ffmpeg_zip -j $NUMBER_OF_NINJA_PROCESSES + autoninja -C out/ffmpeg electron:electron_ffmpeg_zip -j $NUMBER_OF_NINJA_PROCESSES step-verify-mksnapshot: &step-verify-mksnapshot run: @@ -750,26 +737,13 @@ step-setup-linux-for-headless-testing: &step-setup-linux-for-headless-testing sh -e /etc/init.d/xvfb start fi -step-show-goma-stats: &step-show-goma-stats - run: - shell: /bin/bash - name: Check goma stats after build - command: | - set +e - set +o pipefail - python3 $GOMA_DIR/goma_ctl.py stat - python3 $GOMA_DIR/diagnose_goma_log.py - true - when: always - background: true - step-mksnapshot-build: &step-mksnapshot-build run: name: mksnapshot build no_output_timeout: 30m command: | cd src - ninja -C out/Default electron:electron_mksnapshot -j $NUMBER_OF_NINJA_PROCESSES + autoninja -C out/Default electron:electron_mksnapshot -j $NUMBER_OF_NINJA_PROCESSES gn desc out/Default v8:run_mksnapshot_default args > out/Default/mksnapshot_args # Remove unused args from mksnapshot_args SEDOPTION="-i" @@ -792,7 +766,7 @@ step-mksnapshot-build: &step-mksnapshot-build fi fi if [ "$SKIP_DIST_ZIP" != "1" ]; then - ninja -C out/Default electron:electron_mksnapshot_zip -j $NUMBER_OF_NINJA_PROCESSES + autoninja -C out/Default electron:electron_mksnapshot_zip -j $NUMBER_OF_NINJA_PROCESSES (cd out/Default; zip mksnapshot.zip mksnapshot_args gen/v8/embedded.S) fi @@ -802,7 +776,7 @@ step-hunspell-build: &step-hunspell-build command: | cd src if [ "$SKIP_DIST_ZIP" != "1" ]; then - ninja -C out/Default electron:hunspell_dictionaries_zip -j $NUMBER_OF_NINJA_PROCESSES + autoninja -C out/Default electron:hunspell_dictionaries_zip -j $NUMBER_OF_NINJA_PROCESSES fi step-maybe-generate-libcxx: &step-maybe-generate-libcxx @@ -811,9 +785,9 @@ step-maybe-generate-libcxx: &step-maybe-generate-libcxx command: | cd src if [ "`uname`" == "Linux" ]; then - ninja -C out/Default electron:libcxx_headers_zip -j $NUMBER_OF_NINJA_PROCESSES - ninja -C out/Default electron:libcxxabi_headers_zip -j $NUMBER_OF_NINJA_PROCESSES - ninja -C out/Default electron:libcxx_objects_zip -j $NUMBER_OF_NINJA_PROCESSES + autoninja -C out/Default electron:libcxx_headers_zip -j $NUMBER_OF_NINJA_PROCESSES + autoninja -C out/Default electron:libcxxabi_headers_zip -j $NUMBER_OF_NINJA_PROCESSES + autoninja -C out/Default electron:libcxx_objects_zip -j $NUMBER_OF_NINJA_PROCESSES fi step-maybe-generate-breakpad-symbols: &step-maybe-generate-breakpad-symbols @@ -823,7 +797,7 @@ step-maybe-generate-breakpad-symbols: &step-maybe-generate-breakpad-symbols command: | if [ "$GENERATE_SYMBOLS" == "true" ]; then cd src - ninja -C out/Default electron:electron_symbols + autoninja -C out/Default electron:electron_symbols fi step-maybe-zip-symbols: &step-maybe-zip-symbols @@ -832,8 +806,8 @@ step-maybe-zip-symbols: &step-maybe-zip-symbols command: | cd src export BUILD_PATH="$PWD/out/Default" - ninja -C out/Default electron:licenses - ninja -C out/Default electron:electron_version_file + autoninja -C out/Default electron:licenses + autoninja -C out/Default electron:electron_version_file electron/script/zip-symbols.py -b $BUILD_PATH step-maybe-zip-symbols-and-clean: &step-maybe-zip-symbols-and-clean @@ -842,8 +816,8 @@ step-maybe-zip-symbols-and-clean: &step-maybe-zip-symbols-and-clean command: | cd src export BUILD_PATH="$PWD/out/Default" - ninja -C out/Default electron:licenses - ninja -C out/Default electron:electron_version_file + autoninja -C out/Default electron:licenses + autoninja -C out/Default electron:electron_version_file DELETE_DSYMS_AFTER_ZIP=1 electron/script/zip-symbols.py -b $BUILD_PATH step-maybe-cross-arch-snapshot: &step-maybe-cross-arch-snapshot @@ -1190,11 +1164,10 @@ commands: could-be-aks: type: boolean steps: - - *step-setup-goma-for-build + - *step-setup-rbe-for-build - checkout-from-cache: could-be-aks: << parameters.could-be-aks >> - *step-setup-env-for-build - - *step-wait-for-goma - *step-gn-gen-default - *step-gn-check build_and_save_artifacts: @@ -1215,8 +1188,6 @@ commands: - step-electron-dist-build: additional-targets: electron:node_headers third_party/electron_node:overlapped-checker electron:hunspell_dictionaries_zip - - *step-show-goma-stats - # mksnapshot - *step-mksnapshot-build - *step-maybe-cross-arch-snapshot @@ -1345,7 +1316,7 @@ commands: command: | cd src if [ "$SKIP_DIST_ZIP" != "1" ]; then - ninja -C out/Default electron:electron_dist_zip << parameters.additional-targets >> -j $NUMBER_OF_NINJA_PROCESSES + autoninja -C out/Default electron:electron_dist_zip << parameters.additional-targets >> -j $NUMBER_OF_NINJA_PROCESSES if [ "$CHECK_DIST_MANIFEST" == "1" ]; then if [ "`uname`" == "Darwin" ]; then target_os=mac @@ -1446,7 +1417,7 @@ commands: - when: condition: << parameters.build >> steps: - - *step-setup-goma-for-build + - *step-setup-rbe-for-build - when: condition: << parameters.checkout-and-assume-cache >> steps: @@ -1565,7 +1536,6 @@ commands: steps: - *step-depot-tools-add-to-path - *step-setup-env-for-build - - *step-wait-for-goma - *step-get-more-space-on-mac - *step-fix-sync - *step-delete-git-directories @@ -1751,14 +1721,12 @@ commands: - *step-fix-sync - *step-setup-env-for-build - *step-fix-known-hosts-linux - - *step-setup-goma-for-build - - *step-wait-for-goma + - *step-setup-rbe-for-build - *step-gn-gen-default # Electron app - ninja_build_electron: build-type: << parameters.build-type >> - - *step-show-goma-stats - *step-maybe-generate-breakpad-symbols - *step-maybe-electron-dist-strip - step-electron-dist-build diff --git a/DEPS b/DEPS index fdf23eb3333ca..b4cef08f3369b 100644 --- a/DEPS +++ b/DEPS @@ -13,6 +13,8 @@ vars = { '74ab5baccc6f7202c8ac69a8d1e152c29dc1ea76', 'mantle_version': '78d3966b3c331292ea29ec38661b25df0a245948', + 'engflow_reclient_configs_version': + '955335c30a752e9ef7bff375baab5e0819b6c00d', 'pyyaml_version': '3.12', @@ -23,6 +25,7 @@ vars = { 'squirrel_git': 'https://github.com/Squirrel', 'reactiveobjc_git': 'https://github.com/ReactiveCocoa', 'mantle_git': 'https://github.com/Mantle', + 'engflow_git': 'https://github.com/EngFlow', # The path of the sysroots.json file. 'sysroots_json_path': 'electron/script/sysroots.json', @@ -102,6 +105,10 @@ deps = { 'src/third_party/squirrel.mac/vendor/Mantle': { 'url': Var("mantle_git") + '/Mantle.git@' + Var("mantle_version"), 'condition': 'process_deps', + }, + 'src/third_party/engflow-reclient-configs': { + 'url': Var("engflow_git") + '/reclient-configs.git@' + Var("engflow_reclient_configs_version"), + 'condition': 'process_deps' } } diff --git a/appveyor-bake.yml b/appveyor-bake.yml index 6404ac401f2d5..e60bfc9a8ec85 100644 --- a/appveyor-bake.yml +++ b/appveyor-bake.yml @@ -13,7 +13,6 @@ environment: ELECTRON_ENABLE_STACK_DUMPING: 1 MOCHA_REPORTER: mocha-multi-reporters MOCHA_MULTI_REPORTERS: mocha-appveyor-reporter, tap - GOMA_FALLBACK_ON_AUTH_FAILURE: true DEPOT_TOOLS_WIN_TOOLCHAIN: 0 PYTHONIOENCODING: UTF-8 diff --git a/appveyor-woa.yml b/appveyor-woa.yml index 092753b53d7f4..ad487a4fbddda 100644 --- a/appveyor-woa.yml +++ b/appveyor-woa.yml @@ -37,7 +37,6 @@ environment: ELECTRON_ALSO_LOG_TO_STDERR: 1 MOCHA_REPORTER: mocha-multi-reporters MOCHA_MULTI_REPORTERS: "@marshallofsound/mocha-appveyor-reporter, tap" - GOMA_FALLBACK_ON_AUTH_FAILURE: true DEPOT_TOOLS_WIN_TOOLCHAIN: 1 DEPOT_TOOLS_WIN_TOOLCHAIN_BASE_URL: "https://dev-cdn.electronjs.org/windows-toolchains/_" GYP_MSVS_HASH_27370823e7: 28622d16b1 @@ -101,31 +100,20 @@ for: if (Test-Path -Path "$pwd\src\electron") { Remove-Item -Recurse -Force $pwd\src\electron } - - ps: >- - if (Test-Path 'env:RAW_GOMA_AUTH') { - $env:GOMA_OAUTH2_CONFIG_FILE = "$pwd\.goma_oauth2_config" - $env:RAW_GOMA_AUTH | Set-Content $env:GOMA_OAUTH2_CONFIG_FILE - } - git clone https://github.com/electron/build-tools.git - cd build-tools - - npm install + - npx yarn --ignore-engines - mkdir third_party - ps: >- - node -e "require('./src/utils/goma.js').downloadAndPrepare({ gomaOneForAll: true })" - - ps: $env:GN_GOMA_FILE = node -e "console.log(require('./src/utils/goma.js').gnFilePath)" - - ps: $env:LOCAL_GOMA_DIR = node -e "console.log(require('./src/utils/goma.js').dir)" - - cd ..\.. - - ps: .\src\electron\script\start-goma.ps1 -gomaDir $env:LOCAL_GOMA_DIR + node -e "require('./src/utils/reclient.js').downloadAndPrepare({})" + - ps: $env:RECLIENT_HELPER = node -p "require('./src/utils/reclient.js').helperPath" - ps: >- - if (Test-Path 'env:RAW_GOMA_AUTH') { - $goma_login = python3 $env:LOCAL_GOMA_DIR\goma_auth.py info - if ($goma_login -eq 'Login as Fermi Planck') { - Write-warning "Goma authentication is correct"; - } else { - Write-warning "WARNING!!!!!! Goma authentication is incorrect; please update Goma auth token."; - $host.SetShouldExit(1) - } - } + & $env:RECLIENT_HELPER login + - ps: >- + $env:RBE_service = node -e "console.log(require('./src/utils/reclient.js').serviceAddress)" + - ps: >- + $env:RBE_experimental_credentials_helper = $env:RECLIENT_HELPER + - cd ..\.. - ps: $env:CHROMIUM_BUILDTOOLS_PATH="$pwd\src\buildtools" - ps: >- if ($env:GN_CONFIG -ne 'release') { @@ -147,27 +135,26 @@ for: - cd src - ps: $env:PATH="$pwd\third_party\ninja;$env:PATH" - set BUILD_CONFIG_PATH=//electron/build/args/%GN_CONFIG%.gn - - gn gen out/Default "--args=import(\"%BUILD_CONFIG_PATH%\") import(\"%GN_GOMA_FILE%\") %GN_EXTRA_ARGS% " + - gn gen out/Default "--args=import(\"%BUILD_CONFIG_PATH%\") use_remoteexec=true %GN_EXTRA_ARGS% " - gn check out/Default //electron:electron_lib - gn check out/Default //electron:electron_app - gn check out/Default //electron/shell/common/api:mojo - - if DEFINED GN_GOMA_FILE (ninja -j 300 -C out/Default electron:electron_app) else (ninja -C out/Default electron:electron_app) + - if DEFINED ELECTRON_RBE_JWT (autoninja -j 300 -C out/Default electron:electron_app) else (autoninja -C out/Default electron:electron_app) - if "%GN_CONFIG%"=="testing" ( python C:\depot_tools\post_build_ninja_summary.py -C out\Default ) - - gn gen out/ffmpeg "--args=import(\"//electron/build/args/ffmpeg.gn\") %GN_EXTRA_ARGS%" - - ninja -C out/ffmpeg electron:electron_ffmpeg_zip - - ninja -C out/Default electron:electron_dist_zip + - gn gen out/ffmpeg "--args=import(\"//electron/build/args/ffmpeg.gn\") use_remoteexec=true %GN_EXTRA_ARGS%" + - autoninja -C out/ffmpeg electron:electron_ffmpeg_zip + - autoninja -C out/Default electron:electron_dist_zip - gn desc out/Default v8:run_mksnapshot_default args > out/Default/default_mksnapshot_args # Remove unused args from mksnapshot_args - ps: >- Get-Content out/Default/default_mksnapshot_args | Where-Object { -not $_.Contains('--turbo-profiling-input') -And -not $_.Contains('builtins-pgo') -And -not $_.Contains('The gn arg use_goma=true') } | Set-Content out/Default/mksnapshot_args - - ninja -C out/Default electron:electron_mksnapshot_zip + - autoninja -C out/Default electron:electron_mksnapshot_zip - cd out\Default - 7z a mksnapshot.zip mksnapshot_args gen\v8\embedded.S - cd ..\.. - - ninja -C out/Default electron:hunspell_dictionaries_zip - - ninja -C out/Default electron:electron_chromedriver_zip - - ninja -C out/Default electron:node_headers - - python3 %LOCAL_GOMA_DIR%\goma_ctl.py stat + - autoninja -C out/Default electron:hunspell_dictionaries_zip + - autoninja -C out/Default electron:electron_chromedriver_zip + - autoninja -C out/Default electron:node_headers - ps: >- Get-CimInstance -Namespace root\cimv2 -Class Win32_product | Select vendor, description, @{l='install_location';e='InstallLocation'}, @{l='install_date';e='InstallDate'}, @{l='install_date_2';e='InstallDate2'}, caption, version, name, @{l='sku_number';e='SKUNumber'} | ConvertTo-Json | Out-File -Encoding utf8 -FilePath .\installed_software.json - python3 electron/build/profile_toolchain.py --output-json=out/Default/windows_toolchain_profile.json @@ -177,7 +164,7 @@ for: if ($env:GN_CONFIG -eq 'release') { # Needed for msdia140.dll on 64-bit windows $env:Path += ";$pwd\third_party\llvm-build\Release+Asserts\bin" - ninja -C out/Default electron:electron_symbols + autoninja -C out/Default electron:electron_symbols } - ps: >- if ($env:GN_CONFIG -eq 'release') { diff --git a/appveyor.yml b/appveyor.yml index d02f4ee071178..682a5f56006a3 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -37,7 +37,6 @@ environment: ELECTRON_ALSO_LOG_TO_STDERR: 1 MOCHA_REPORTER: mocha-multi-reporters MOCHA_MULTI_REPORTERS: "@marshallofsound/mocha-appveyor-reporter, tap" - GOMA_FALLBACK_ON_AUTH_FAILURE: true DEPOT_TOOLS_WIN_TOOLCHAIN: 1 DEPOT_TOOLS_WIN_TOOLCHAIN_BASE_URL: "https://dev-cdn.electronjs.org/windows-toolchains/_" GYP_MSVS_HASH_27370823e7: 28622d16b1 @@ -99,31 +98,20 @@ for: if (Test-Path -Path "$pwd\src\electron") { Remove-Item -Recurse -Force $pwd\src\electron } - - ps: >- - if (Test-Path 'env:RAW_GOMA_AUTH') { - $env:GOMA_OAUTH2_CONFIG_FILE = "$pwd\.goma_oauth2_config" - $env:RAW_GOMA_AUTH | Set-Content $env:GOMA_OAUTH2_CONFIG_FILE - } - git clone https://github.com/electron/build-tools.git - cd build-tools - - npm install + - npx yarn --ignore-engines - mkdir third_party - ps: >- - node -e "require('./src/utils/goma.js').downloadAndPrepare({ gomaOneForAll: true })" - - ps: $env:GN_GOMA_FILE = node -e "console.log(require('./src/utils/goma.js').gnFilePath)" - - ps: $env:LOCAL_GOMA_DIR = node -e "console.log(require('./src/utils/goma.js').dir)" - - cd ..\.. - - ps: .\src\electron\script\start-goma.ps1 -gomaDir $env:LOCAL_GOMA_DIR + node -e "require('./src/utils/reclient.js').downloadAndPrepare({})" + - ps: $env:RECLIENT_HELPER = node -p "require('./src/utils/reclient.js').helperPath" - ps: >- - if (Test-Path 'env:RAW_GOMA_AUTH') { - $goma_login = python3 $env:LOCAL_GOMA_DIR\goma_auth.py info - if ($goma_login -eq 'Login as Fermi Planck') { - Write-warning "Goma authentication is correct"; - } else { - Write-warning "WARNING!!!!!! Goma authentication is incorrect; please update Goma auth token."; - $host.SetShouldExit(1) - } - } + & $env:RECLIENT_HELPER login + - ps: >- + $env:RBE_service = node -e "console.log(require('./src/utils/reclient.js').serviceAddress)" + - ps: >- + $env:RBE_experimental_credentials_helper = $env:RECLIENT_HELPER + - cd ..\.. - ps: $env:CHROMIUM_BUILDTOOLS_PATH="$pwd\src\buildtools" - ps: >- if ($env:GN_CONFIG -ne 'release') { @@ -145,27 +133,26 @@ for: - cd src - ps: $env:PATH="$pwd\third_party\ninja;$env:PATH" - set BUILD_CONFIG_PATH=//electron/build/args/%GN_CONFIG%.gn - - gn gen out/Default "--args=import(\"%BUILD_CONFIG_PATH%\") import(\"%GN_GOMA_FILE%\") %GN_EXTRA_ARGS% " + - gn gen out/Default "--args=import(\"%BUILD_CONFIG_PATH%\") use_remoteexec=true %GN_EXTRA_ARGS% " - gn check out/Default //electron:electron_lib - gn check out/Default //electron:electron_app - gn check out/Default //electron/shell/common/api:mojo - - if DEFINED GN_GOMA_FILE (ninja -j 300 -C out/Default electron:electron_app) else (ninja -C out/Default electron:electron_app) + - if DEFINED ELECTRON_RBE_JWT (autoninja -j 300 -C out/Default electron:electron_app) else (autoninja -C out/Default electron:electron_app) - if "%GN_CONFIG%"=="testing" ( python C:\depot_tools\post_build_ninja_summary.py -C out\Default ) - - gn gen out/ffmpeg "--args=import(\"//electron/build/args/ffmpeg.gn\") %GN_EXTRA_ARGS%" - - ninja -C out/ffmpeg electron:electron_ffmpeg_zip - - ninja -C out/Default electron:electron_dist_zip + - gn gen out/ffmpeg "--args=import(\"//electron/build/args/ffmpeg.gn\") use_remoteexec=true %GN_EXTRA_ARGS%" + - autoninja -C out/ffmpeg electron:electron_ffmpeg_zip + - autoninja -C out/Default electron:electron_dist_zip - gn desc out/Default v8:run_mksnapshot_default args > out/Default/default_mksnapshot_args # Remove unused args from mksnapshot_args - ps: >- Get-Content out/Default/default_mksnapshot_args | Where-Object { -not $_.Contains('--turbo-profiling-input') -And -not $_.Contains('builtins-pgo') -And -not $_.Contains('The gn arg use_goma=true') } | Set-Content out/Default/mksnapshot_args - - ninja -C out/Default electron:electron_mksnapshot_zip + - autoninja -C out/Default electron:electron_mksnapshot_zip - cd out\Default - 7z a mksnapshot.zip mksnapshot_args gen\v8\embedded.S - cd ..\.. - - ninja -C out/Default electron:hunspell_dictionaries_zip - - ninja -C out/Default electron:electron_chromedriver_zip - - ninja -C out/Default electron:node_headers - - python3 %LOCAL_GOMA_DIR%\goma_ctl.py stat + - autoninja -C out/Default electron:hunspell_dictionaries_zip + - autoninja -C out/Default electron:electron_chromedriver_zip + - autoninja -C out/Default electron:node_headers - ps: >- Get-CimInstance -Namespace root\cimv2 -Class Win32_product | Select vendor, description, @{l='install_location';e='InstallLocation'}, @{l='install_date';e='InstallDate'}, @{l='install_date_2';e='InstallDate2'}, caption, version, name, @{l='sku_number';e='SKUNumber'} | ConvertTo-Json | Out-File -Encoding utf8 -FilePath .\installed_software.json - python3 electron/build/profile_toolchain.py --output-json=out/Default/windows_toolchain_profile.json @@ -174,7 +161,7 @@ for: if ($env:GN_CONFIG -eq 'release') { # Needed for msdia140.dll on 64-bit windows $env:Path += ";$pwd\third_party\llvm-build\Release+Asserts\bin" - ninja -C out/Default electron:electron_symbols + autoninja -C out/Default electron:electron_symbols } - ps: >- if ($env:GN_CONFIG -eq 'release') { diff --git a/patches/chromium/.patches b/patches/chromium/.patches index 19a7702967437..d8a6d48e4f99f 100644 --- a/patches/chromium/.patches +++ b/patches/chromium/.patches @@ -129,3 +129,4 @@ chore_remove_check_is_test_on_script_injection_tracker.patch fix_restore_original_resize_performance_on_macos.patch feat_allow_code_cache_in_custom_schemes.patch enable_partition_alloc_ref_count_size.patch +build_run_reclient_cfg_generator_after_chrome.patch diff --git a/patches/chromium/build_run_reclient_cfg_generator_after_chrome.patch b/patches/chromium/build_run_reclient_cfg_generator_after_chrome.patch new file mode 100644 index 0000000000000..66dce9e49da5d --- /dev/null +++ b/patches/chromium/build_run_reclient_cfg_generator_after_chrome.patch @@ -0,0 +1,30 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: Samuel Attard +Date: Thu, 28 Dec 2023 17:09:08 +1300 +Subject: build: run reclient cfg generator after chrome + +We can't run our generator after chromes in gclient hooks so instead we patch +their generator to run ours immediately after. + +This can't be upstreamed though may be replaceable later with some upstream refactors +around reclient config generation. + +diff --git a/buildtools/reclient_cfgs/configure_reclient_cfgs.py b/buildtools/reclient_cfgs/configure_reclient_cfgs.py +index 35aaf829ae998b52ca2ae21450a66f2c53fd457c..ecd46ccb33c5a8fe1cae526081a3bd1aa997cf81 100755 +--- a/buildtools/reclient_cfgs/configure_reclient_cfgs.py ++++ b/buildtools/reclient_cfgs/configure_reclient_cfgs.py +@@ -317,4 +317,13 @@ def main(): + + + if __name__ == "__main__": +- sys.exit(main()) ++ r = main() ++ if r != 0: ++ sys.exit(r) ++ ++ electron_script = os.path.join(CHROMIUM_SRC, 'third_party', 'engflow-reclient-configs', 'configure_reclient.py') ++ r = subprocess.call([sys.executable, electron_script, '--src_dir=src', '--force']) ++ if r != 0: ++ sys.exit(r) ++ ++ print('done') diff --git a/patches/config.json b/patches/config.json index db929cb196534..4855e2e03d95a 100644 --- a/patches/config.json +++ b/patches/config.json @@ -21,5 +21,7 @@ "src/electron/patches/ReactiveObjC": "src/third_party/squirrel.mac/vendor/ReactiveObjC", - "src/electron/patches/webrtc": "src/third_party/webrtc" + "src/electron/patches/webrtc": "src/third_party/webrtc", + + "src/electron/patches/reclient-configs": "src/third_party/engflow-reclient-configs" } diff --git a/patches/reclient-configs/.patches b/patches/reclient-configs/.patches new file mode 100644 index 0000000000000..cf8760bb8ede4 --- /dev/null +++ b/patches/reclient-configs/.patches @@ -0,0 +1,3 @@ +fix_disable_compression_threshold.patch +fix_add_python_remote_wrapper.patch +build_on_windows_provide_clang_wrapper_as_toolchain_input.patch diff --git a/patches/reclient-configs/build_on_windows_provide_clang_wrapper_as_toolchain_input.patch b/patches/reclient-configs/build_on_windows_provide_clang_wrapper_as_toolchain_input.patch new file mode 100644 index 0000000000000..a4195cc727959 --- /dev/null +++ b/patches/reclient-configs/build_on_windows_provide_clang_wrapper_as_toolchain_input.patch @@ -0,0 +1,19 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: Samuel Attard +Date: Tue, 9 Jan 2024 11:18:28 +1300 +Subject: build: on windows provide clang wrapper as toolchain input + +Should be upstreamed, fixes the executable bit being missing on win->lin x builds + +diff --git a/chromium-browser-clang/rewrapper_windows.cfg b/chromium-browser-clang/rewrapper_windows.cfg +index 543b68bbd736bfb6cb2cf3e939c4cdc34b6a18da..539f37a53d990df39203496bb2d579f2ce38de75 100644 +--- a/chromium-browser-clang/rewrapper_windows.cfg ++++ b/chromium-browser-clang/rewrapper_windows.cfg +@@ -15,6 +15,5 @@ + # This config is merged with Chromium config. See README.md. + + server_address=pipe://reproxy.pipe +-inputs={src_dir}/buildtools/reclient_cfgs/chromium-browser-clang/clang_remote_wrapper +-toolchain_inputs={linux_clang_base_path}/bin/clang ++toolchain_inputs={linux_clang_base_path}/bin/clang,{src_dir}/buildtools/reclient_cfgs/chromium-browser-clang/clang_remote_wrapper + remote_wrapper={src_dir}/buildtools/reclient_cfgs/chromium-browser-clang/clang_remote_wrapper diff --git a/patches/reclient-configs/fix_add_python_remote_wrapper.patch b/patches/reclient-configs/fix_add_python_remote_wrapper.patch new file mode 100644 index 0000000000000..0c30b8490ecaa --- /dev/null +++ b/patches/reclient-configs/fix_add_python_remote_wrapper.patch @@ -0,0 +1,144 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: Samuel Attard +Date: Thu, 28 Dec 2023 16:47:13 +1300 +Subject: fix: add python_remote_wrapper + +Our RBE cluster struggles to find python3 but bash finds it when spawned +so we wrap all python remote execs with a helper bash script that just +runs the underlying command. + +diff --git a/configure_reclient.py b/configure_reclient.py +index 5948be95b6b19e6af4f9a9503c18ce2bebf63f2d..a157ceeb155987f6ab9b11d3cef8c2c9894e0436 100755 +--- a/configure_reclient.py ++++ b/configure_reclient.py +@@ -21,6 +21,7 @@ import os + import re + import runpy + import shutil ++import stat + import string + import subprocess + import sys +@@ -109,6 +110,8 @@ class ReclientConfigurator: + self.download_linux_clang_toolchain() + self.generate_clang_remote_wrapper() + ++ self.generate_python_remote_wrapper() ++ + # Reproxy config includes auth and network-related parameters. + self.generate_reproxy_cfg() + # Rewrapper configs describe how different tools should be run remotely. +@@ -199,6 +202,32 @@ class ReclientConfigurator: + (f'{Paths.src_dir}/buildtools/reclient_cfgs/chromium-browser-clang/' + 'clang_remote_wrapper'), clang_remote_wrapper) + ++ FileUtils.chmod_x((f'{Paths.src_dir}/buildtools/reclient_cfgs/chromium-browser-clang/' ++ 'clang_remote_wrapper')) ++ ++ @staticmethod ++ def generate_python_remote_wrapper(): ++ # Load python remote wrapper template. ++ template_file = (f'{Paths.script_dir}/python/' ++ 'python_remote_wrapper.template') ++ python_remote_wrapper_template = FileUtils.read_text_file(template_file) ++ ++ # Variables to set in the template. ++ template_vars = { ++ } ++ ++ # Substitute variables into the template. ++ python_remote_wrapper = ShellTemplate( ++ python_remote_wrapper_template).substitute(template_vars) ++ ++ # Write the python remote wrapper. ++ FileUtils.write_text_file( ++ (f'{Paths.src_dir}/buildtools/reclient_cfgs/python/' ++ 'python_remote_wrapper'), python_remote_wrapper) ++ ++ FileUtils.chmod_x((f'{Paths.src_dir}/buildtools/reclient_cfgs/python/' ++ 'python_remote_wrapper')) ++ + def generate_reproxy_cfg(self): + # Load Chromium config template and remove everything starting with $ + # symbol on each line. +@@ -536,6 +565,11 @@ class FileUtils: + + shutil.move(filepath_new, filepath) + ++ @classmethod ++ def chmod_x(cls, filepath): ++ st = os.stat(filepath) ++ os.chmod(filepath, st.st_mode | stat.S_IEXEC) ++ + @classmethod + def create_generated_header(cls, source_files): + if not isinstance(source_files, (list, tuple)): +diff --git a/python/python_remote_wrapper.template b/python/python_remote_wrapper.template +new file mode 100644 +index 0000000000000000000000000000000000000000..54817e4f6f9e3cb2f1e7ea1317fa8fefdf7a7da5 +--- /dev/null ++++ b/python/python_remote_wrapper.template +@@ -0,0 +1,29 @@ ++#!/bin/bash ++# Copyright (c) 2023 Contributors to the reclient-configs project. All rights reserved. ++# ++# Licensed under the Apache License, Version 2.0 (the "License"); ++# you may not use this file except in compliance with the License. ++# You may obtain a copy of the License at ++# ++# http://www.apache.org/licenses/LICENSE-2.0 ++# ++# Unless required by applicable law or agreed to in writing, software ++# distributed under the License is distributed on an "AS IS" BASIS, ++# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. ++# See the License for the specific language governing permissions and ++# limitations under the License. ++ ++# AUTOGENERATED FILE - DO NOT EDIT ++# Generated by: ++# {configurator_dir}/configure_reclient.py ++# To edit update: ++# {configurator_dir}/python/python_remote_wrapper.template ++# And rerun configurator. ++ ++# WARNING: This file is a part of reclient action inputs. Any modification will ++# invalidate remote cache. ++ ++set -e ++ ++# Launch ++"$1" "${@:2}" +diff --git a/python/rewrapper_linux.cfg b/python/rewrapper_linux.cfg +index 951bc66afd28b280fe936141f0b1a8e473bdba05..4ac7ec4b7559ffc133912ee932f9cf8e7450e4db 100644 +--- a/python/rewrapper_linux.cfg ++++ b/python/rewrapper_linux.cfg +@@ -13,3 +13,6 @@ + # limitations under the License. + + # This config is merged with Chromium config. See README.md. ++ ++inputs=buildtools/reclient_cfgs/python/python_remote_wrapper ++remote_wrapper=../../buildtools/reclient_cfgs/python/python_remote_wrapper +\ No newline at end of file +diff --git a/python/rewrapper_mac.cfg b/python/rewrapper_mac.cfg +index 951bc66afd28b280fe936141f0b1a8e473bdba05..4ac7ec4b7559ffc133912ee932f9cf8e7450e4db 100644 +--- a/python/rewrapper_mac.cfg ++++ b/python/rewrapper_mac.cfg +@@ -13,3 +13,6 @@ + # limitations under the License. + + # This config is merged with Chromium config. See README.md. ++ ++inputs=buildtools/reclient_cfgs/python/python_remote_wrapper ++remote_wrapper=../../buildtools/reclient_cfgs/python/python_remote_wrapper +\ No newline at end of file +diff --git a/python/rewrapper_windows.cfg b/python/rewrapper_windows.cfg +index 7ff80607546455eb7c5f16a410770d18949cd7f5..ce19dc95094d87b494e4b89a2bf1a368ace567a2 100644 +--- a/python/rewrapper_windows.cfg ++++ b/python/rewrapper_windows.cfg +@@ -15,3 +15,5 @@ + # This config is merged with Chromium config. See README.md. + + server_address=pipe://reproxy.pipe ++inputs=buildtools/reclient_cfgs/python/python_remote_wrapper ++remote_wrapper=../../buildtools/reclient_cfgs/python/python_remote_wrapper diff --git a/patches/reclient-configs/fix_disable_compression_threshold.patch b/patches/reclient-configs/fix_disable_compression_threshold.patch new file mode 100644 index 0000000000000..0be4eac477771 --- /dev/null +++ b/patches/reclient-configs/fix_disable_compression_threshold.patch @@ -0,0 +1,17 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: Samuel Attard +Date: Thu, 28 Dec 2023 16:13:18 +1300 +Subject: fix: disable compression_threshold + +This appears to cause issues with nginx ingress for RBE endpoints, disable +it till we can figure out why. + +diff --git a/reproxy.cfg b/reproxy.cfg +index 4325d36cbfcb6d81eb0f8dc7ab7cf8e8857d608c..2390a73f4674a591ef833d10697e4aaf3894c011 100644 +--- a/reproxy.cfg ++++ b/reproxy.cfg +@@ -17,3 +17,4 @@ + # Unset Chromium variables. + service= + automatic_auth= ++compression_threshold= diff --git a/script/prepare-appveyor.js b/script/prepare-appveyor.js index 7bac5a9a2d50e..56e39a5f66632 100644 --- a/script/prepare-appveyor.js +++ b/script/prepare-appveyor.js @@ -110,7 +110,6 @@ async function callAppVeyorBuildJobs (targetBranch, job, options) { ELECTRON_OUT_DIR: 'Default', ELECTRON_ENABLE_STACK_DUMPING: 1, ELECTRON_ALSO_LOG_TO_STDERR: 1, - GOMA_FALLBACK_ON_AUTH_FAILURE: true, DEPOT_TOOLS_WIN_TOOLCHAIN: 0, PYTHONIOENCODING: 'UTF-8' }; diff --git a/script/start-goma.ps1 b/script/start-goma.ps1 deleted file mode 100755 index 20e061105ab2c..0000000000000 --- a/script/start-goma.ps1 +++ /dev/null @@ -1,10 +0,0 @@ -param([string]$gomaDir=$PWD) -$cmdPath = Join-Path -Path $gomaDir -ChildPath "goma_ctl.py" -Start-Process -FilePath cmd -ArgumentList "/C", "python3", "$cmdPath", "ensure_start" -$timedOut = $false; $waitTime = 0; $waitIncrement = 5; $timeout=120; -Do { sleep $waitIncrement; $timedOut = (($waitTime+=$waitIncrement) -gt $timeout); iex "$gomaDir\gomacc.exe port 2" > $null; } Until(($LASTEXITCODE -eq 0) -or $timedOut) -if ($timedOut) { - write-error 'Timed out waiting for goma to start'; exit 1; -} else { - Write-Output "Successfully started goma!" -}