diff --git a/.circleci/.gitignore b/.circleci/.gitignore new file mode 100644 index 0000000000000..2b7d56379cacd --- /dev/null +++ b/.circleci/.gitignore @@ -0,0 +1 @@ +config-staging diff --git a/.circleci/config.yml b/.circleci/config.yml index 344b434438134..adcf9e502490c 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -6,6 +6,7 @@ setup: true # Orbs orbs: path-filtering: circleci/path-filtering@0.1.0 + continuation: circleci/continuation@0.2.0 # All input parameters to pass to build config parameters: @@ -43,103 +44,33 @@ parameters: default: all enum: ["all", "osx-x64", "osx-arm64", "mas-x64", "mas-arm64"] -# Envs -env-global: &env-global - ELECTRON_OUT_DIR: Default - -env-linux-medium: &env-linux-medium - <<: *env-global - NUMBER_OF_NINJA_PROCESSES: 3 - -# Executors -executors: - linux-docker: - parameters: - size: - description: "Docker executor size" - default: 2xlarge+ - type: enum - enum: ["medium", "xlarge", "2xlarge+"] - docker: - - image: ghcr.io/electron/build:27db4a3e3512bfd2e47f58cea69922da0835f1d9 - resource_class: << parameters.size >> - -# List of always run steps -step-checkout-electron: &step-checkout-electron - checkout: - path: src/electron - -steps-lint: &steps-lint - steps: - - *step-checkout-electron - - run: - name: Setup third_party Depot Tools - command: | - # "depot_tools" has to be checkout into "//third_party/depot_tools" so pylint.py can a "pylintrc" file. - git clone https://chromium.googlesource.com/chromium/tools/depot_tools.git src/third_party/depot_tools - echo 'export PATH="$PATH:'"$PWD"'/src/third_party/depot_tools"' >> $BASH_ENV - - run: - name: Download GN Binary - command: | - chromium_revision="$(grep -A1 chromium_version src/electron/DEPS | tr -d '\n' | cut -d\' -f4)" - gn_version="$(curl -sL "https://chromium.googlesource.com/chromium/src/+/${chromium_revision}/DEPS?format=TEXT" | base64 -d | grep gn_version | head -n1 | cut -d\' -f4)" - - cipd ensure -ensure-file - -root . \<<-CIPD - \$ServiceURL https://chrome-infra-packages.appspot.com/ - @Subdir src/buildtools/linux64 - gn/gn/linux-amd64 $gn_version - CIPD - - echo 'export CHROMIUM_BUILDTOOLS_PATH="'"$PWD"'/src/buildtools"' >> $BASH_ENV - - run: - name: Download clang-format Binary - command: | - chromium_revision="$(grep -A1 chromium_version src/electron/DEPS | tr -d '\n' | cut -d\' -f4)" - - sha1_path='buildtools/linux64/clang-format.sha1' - curl -sL "https://chromium.googlesource.com/chromium/src/+/${chromium_revision}/${sha1_path}?format=TEXT" | base64 -d > "src/${sha1_path}" - - download_from_google_storage.py --no_resume --no_auth --bucket chromium-clang-format -s "src/${sha1_path}" - - run: - name: Run Lint - command: | - # gn.py tries to find a gclient root folder starting from the current dir. - # When it fails and returns "None" path, the whole script fails. Let's "fix" it. - touch .gclient - # Another option would be to checkout "buildtools" inside the Electron checkout, - # but then we would lint its contents (at least gn format), and it doesn't pass it. - - cd src/electron - node script/yarn install --frozen-lockfile - node script/yarn lint - - run: - name: Run Script Typechecker - command: | - cd src/electron - node script/yarn tsc -p tsconfig.script.json - -# List of always run jobs. jobs: - lint: - executor: - name: linux-docker - size: medium - environment: - <<: *env-linux-medium - <<: *steps-lint - -# Initial setup workflow -workflows: - lint: - jobs: - # Job inherited from path-filtering orb - - path-filtering/filter: + generate-config: + docker: + - image: cimg/node:16.14 + steps: + - checkout + - path-filtering/set-parameters: base-revision: main - # Params for mapping; `path-to-test parameter-to-set value-for-parameter` for each row mapping: | ^((?!docs/).)*$ run-build-mac true ^((?!docs/).)*$ run-build-linux true docs/.* run-docs-only true ^((?!docs/).)*$ run-docs-only false - config-path: .circleci/build_config.yml - - lint + - run: + command: | + cd .circleci/config + yarn + export CIRCLECI_BINARY="$HOME/circleci" + curl -fLSs https://raw.githubusercontent.com/CircleCI-Public/circleci-cli/master/install.sh | DESTDIR=$CIRCLECI_BINARY bash + node build.js + name: Pack config.yml + - continuation/continue: + configuration_path: .circleci/config-staging/built.yml + parameters: /tmp/pipeline-parameters.json + +# Initial setup workflow +workflows: + setup: + jobs: + - generate-config diff --git a/.circleci/build_config.yml b/.circleci/config/base.yml similarity index 97% rename from .circleci/build_config.yml rename to .circleci/config/base.yml index d66ece9a80bac..a9f5ba36b3b61 100644 --- a/.circleci/build_config.yml +++ b/.circleci/config/base.yml @@ -927,12 +927,12 @@ step-touch-sync-done: &step-touch-sync-done step-maybe-restore-src-cache: &step-maybe-restore-src-cache restore_cache: keys: - - v12-src-cache-{{ checksum "src/electron/.depshash" }} + - v14-src-cache-{{ checksum "src/electron/.depshash" }} name: Restoring src cache step-maybe-restore-src-cache-marker: &step-maybe-restore-src-cache-marker restore_cache: keys: - - v5-src-cache-marker-{{ checksum "src/electron/.depshash" }} + - v14-src-cache-marker-{{ checksum "src/electron/.depshash" }} name: Restoring src cache marker # Restore exact or closest git cache based on the hash of DEPS and .circle-sync-done @@ -941,10 +941,10 @@ step-maybe-restore-src-cache-marker: &step-maybe-restore-src-cache-marker step-maybe-restore-git-cache: &step-maybe-restore-git-cache restore_cache: paths: - - gclient-cache + - git-cache keys: - - v5-gclient-cache-{{ checksum "src/electron/.circle-sync-done" }}-{{ checksum "src/electron/DEPS" }} - - v5-gclient-cache-{{ checksum "src/electron/.circle-sync-done" }} + - v1-git-cache-{{ checksum "src/electron/.circle-sync-done" }}-{{ checksum "src/electron/DEPS" }} + - v1-git-cache-{{ checksum "src/electron/.circle-sync-done" }} name: Conditionally restoring git cache step-restore-out-cache: &step-restore-out-cache @@ -961,15 +961,15 @@ step-set-git-cache-path: &step-set-git-cache-path command: | # CircleCI does not support interpolation when setting environment variables. # https://circleci.com/docs/2.0/env-vars/#setting-an-environment-variable-in-a-shell-command - echo 'export GIT_CACHE_PATH="$PWD/gclient-cache"' >> $BASH_ENV + echo 'export GIT_CACHE_PATH="$PWD/git-cache"' >> $BASH_ENV # Persist the git cache based on the hash of DEPS and .circle-sync-done # If the src cache was restored above then this will persist an empty cache step-save-git-cache: &step-save-git-cache save_cache: paths: - - gclient-cache - key: v5-gclient-cache-{{ checksum "src/electron/.circle-sync-done" }}-{{ checksum "src/electron/DEPS" }} + - git-cache + key: v1-git-cache-{{ checksum "src/electron/.circle-sync-done" }}-{{ checksum "src/electron/DEPS" }} name: Persisting git cache step-save-out-cache: &step-save-out-cache @@ -1014,7 +1014,7 @@ step-save-src-cache: &step-save-src-cache save_cache: paths: - /var/portal - key: v12-src-cache-{{ checksum "/var/portal/src/electron/.depshash" }} + key: v14-src-cache-{{ checksum "/var/portal/src/electron/.depshash" }} name: Persisting src cache step-make-src-cache-marker: &step-make-src-cache-marker run: @@ -1024,7 +1024,7 @@ step-save-src-cache-marker: &step-save-src-cache-marker save_cache: paths: - .src-cache-marker - key: v5-src-cache-marker-{{ checksum "/var/portal/src/electron/.depshash" }} + key: v14-src-cache-marker-{{ checksum "/var/portal/src/electron/.depshash" }} step-maybe-early-exit-no-doc-change: &step-maybe-early-exit-no-doc-change run: @@ -1401,12 +1401,40 @@ commands: - *step-checkout-electron - *step-run-electron-only-hooks - *step-generate-deps-hash-cleanly - - *step-mark-sync-done - # Save git cache AFTER sync marked done because other jobs expect that to be the case - when: condition: << parameters.save-git-cache >> steps: - - *step-save-git-cache + - *step-save-git-cache + # Mark sync as done _after_ saving the git cache so that it is uploaded + # only when the src cache was not present + # Their are theoretically two cases for this cache key + # 1. `vX-git-cache-DONE-{deps_hash} + # 2. `vX-git-cache-EMPTY-{deps_hash} + # + # Case (1) occurs when the flag file has "DONE" in it + # which only occurs when "step-mark-sync-done" is run + # or when the src cache was restored successfully as that + # flag file contains "DONE" in the src cache. + # + # Case (2) occurs when the flag file is empty, this occurs + # when the src cache was not restored and "step-mark-sync-done" + # has not run yet. + # + # Notably both of these cases also have completely different + # gclient cache states. + # In (1) the git cache is completely empty as we didn't run + # "gclient sync" because the src cache was restored. + # In (2) the git cache is full as we had to run "gclient sync" + # + # This allows us to do make the follow transitive assumption: + # In cases where the src cache is restored, saving the git cache + # will save an empty cache. In cases where the src cache is built + # during this build the git cache will save a full cache. + # + # In order words if there is a src cache for a given DEPS hash + # the git cache restored will be empty. But if the src cache + # is missing we will restore a useful git cache. + - *step-mark-sync-done - *step-minimize-workspace-size-from-checkout - *step-delete-git-directories - when: @@ -2475,3 +2503,6 @@ workflows: ignore: /pull\/[0-9]+/ requires: - mas-testing-arm64 + lint: + jobs: + - lint diff --git a/.circleci/config/build.js b/.circleci/config/build.js new file mode 100644 index 0000000000000..4e255608a0905 --- /dev/null +++ b/.circleci/config/build.js @@ -0,0 +1,34 @@ +const cp = require('child_process'); +const fs = require('fs-extra'); +const path = require('path'); +const yaml = require('js-yaml'); + +const STAGING_DIR = path.resolve(__dirname, '..', 'config-staging'); + +function copyAndExpand(dir = './') { + const absDir = path.resolve(__dirname, dir); + const targetDir = path.resolve(STAGING_DIR, dir); + + if (!fs.existsSync(targetDir)) { + fs.mkdirSync(targetDir); + } + + for (const file of fs.readdirSync(absDir)) { + if (!file.endsWith('.yml')) { + if (fs.statSync(path.resolve(absDir, file)).isDirectory()) { + copyAndExpand(path.join(dir, file)); + } + continue; + } + + fs.writeFileSync(path.resolve(targetDir, file), yaml.dump(yaml.load(fs.readFileSync(path.resolve(absDir, file), 'utf8')), { + noRefs: true, + })); + } +} + +if (fs.pathExists(STAGING_DIR)) fs.removeSync(STAGING_DIR); +copyAndExpand(); + +const output = cp.spawnSync(process.env.CIRCLECI_BINARY || 'circleci', ['config', 'pack', STAGING_DIR]); +fs.writeFileSync(path.resolve(STAGING_DIR, 'built.yml'), output.stdout.toString()); diff --git a/.circleci/config/jobs/lint.yml b/.circleci/config/jobs/lint.yml new file mode 100644 index 0000000000000..2137566547120 --- /dev/null +++ b/.circleci/config/jobs/lint.yml @@ -0,0 +1,51 @@ +executor: + name: linux-docker + size: medium +steps: + - checkout: + path: src/electron + - run: + name: Setup third_party Depot Tools + command: | + # "depot_tools" has to be checkout into "//third_party/depot_tools" so pylint.py can a "pylintrc" file. + git clone https://chromium.googlesource.com/chromium/tools/depot_tools.git src/third_party/depot_tools + echo 'export PATH="$PATH:'"$PWD"'/src/third_party/depot_tools"' >> $BASH_ENV + - run: + name: Download GN Binary + command: | + chromium_revision="$(grep -A1 chromium_version src/electron/DEPS | tr -d '\n' | cut -d\' -f4)" + gn_version="$(curl -sL "https://chromium.googlesource.com/chromium/src/+/${chromium_revision}/DEPS?format=TEXT" | base64 -d | grep gn_version | head -n1 | cut -d\' -f4)" + + cipd ensure -ensure-file - -root . \<<-CIPD + \$ServiceURL https://chrome-infra-packages.appspot.com/ + @Subdir src/buildtools/linux64 + gn/gn/linux-amd64 $gn_version + CIPD + + echo 'export CHROMIUM_BUILDTOOLS_PATH="'"$PWD"'/src/buildtools"' >> $BASH_ENV + - run: + name: Download clang-format Binary + command: | + chromium_revision="$(grep -A1 chromium_version src/electron/DEPS | tr -d '\n' | cut -d\' -f4)" + + sha1_path='buildtools/linux64/clang-format.sha1' + curl -sL "https://chromium.googlesource.com/chromium/src/+/${chromium_revision}/${sha1_path}?format=TEXT" | base64 -d > "src/${sha1_path}" + + download_from_google_storage.py --no_resume --no_auth --bucket chromium-clang-format -s "src/${sha1_path}" + - run: + name: Run Lint + command: | + # gn.py tries to find a gclient root folder starting from the current dir. + # When it fails and returns "None" path, the whole script fails. Let's "fix" it. + touch .gclient + # Another option would be to checkout "buildtools" inside the Electron checkout, + # but then we would lint its contents (at least gn format), and it doesn't pass it. + + cd src/electron + node script/yarn install --frozen-lockfile + node script/yarn lint + - run: + name: Run Script Typechecker + command: | + cd src/electron + node script/yarn tsc -p tsconfig.script.json diff --git a/.circleci/config/package.json b/.circleci/config/package.json new file mode 100644 index 0000000000000..054dd7c11cc9e --- /dev/null +++ b/.circleci/config/package.json @@ -0,0 +1,10 @@ +{ + "name": "@electron/circleci-config", + "version": "0.0.0", + "private": true, + "license": "MIT", + "dependencies": { + "fs-extra": "^10.1.0", + "js-yaml": "^4.1.0" + } +} diff --git a/.circleci/config/yarn.lock b/.circleci/config/yarn.lock new file mode 100644 index 0000000000000..51b2d9877643f --- /dev/null +++ b/.circleci/config/yarn.lock @@ -0,0 +1,43 @@ +# THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY. +# yarn lockfile v1 + + +argparse@^2.0.1: + version "2.0.1" + resolved "https://registry.yarnpkg.com/argparse/-/argparse-2.0.1.tgz#246f50f3ca78a3240f6c997e8a9bd1eac49e4b38" + integrity sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q== + +fs-extra@^10.1.0: + version "10.1.0" + resolved "https://registry.yarnpkg.com/fs-extra/-/fs-extra-10.1.0.tgz#02873cfbc4084dde127eaa5f9905eef2325d1abf" + integrity sha512-oRXApq54ETRj4eMiFzGnHWGy+zo5raudjuxN0b8H7s/RU2oW0Wvsx9O0ACRN/kRq9E8Vu/ReskGB5o3ji+FzHQ== + dependencies: + graceful-fs "^4.2.0" + jsonfile "^6.0.1" + universalify "^2.0.0" + +graceful-fs@^4.1.6, graceful-fs@^4.2.0: + version "4.2.10" + resolved "https://registry.yarnpkg.com/graceful-fs/-/graceful-fs-4.2.10.tgz#147d3a006da4ca3ce14728c7aefc287c367d7a6c" + integrity sha512-9ByhssR2fPVsNZj478qUUbKfmL0+t5BDVyjShtyZZLiK7ZDAArFFfopyOTj0M05wE2tJPisA4iTnnXl2YoPvOA== + +js-yaml@^4.1.0: + version "4.1.0" + resolved "https://registry.yarnpkg.com/js-yaml/-/js-yaml-4.1.0.tgz#c1fb65f8f5017901cdd2c951864ba18458a10602" + integrity sha512-wpxZs9NoxZaJESJGIZTyDEaYpl0FKSA+FB9aJiyemKhMwkxQg63h4T1KJgUGHpTqPDNRcmmYLugrRjJlBtWvRA== + dependencies: + argparse "^2.0.1" + +jsonfile@^6.0.1: + version "6.1.0" + resolved "https://registry.yarnpkg.com/jsonfile/-/jsonfile-6.1.0.tgz#bc55b2634793c679ec6403094eb13698a6ec0aae" + integrity sha512-5dgndWOriYSm5cnYaJNhalLNDKOqFwyDB/rr1E9ZsGciGvKPs8R2xYGCacuf3z6K1YKDz182fd+fY3cn3pMqXQ== + dependencies: + universalify "^2.0.0" + optionalDependencies: + graceful-fs "^4.1.6" + +universalify@^2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/universalify/-/universalify-2.0.0.tgz#75a4984efedc4b08975c5aeb73f530d02df25717" + integrity sha512-hAZsKq7Yy11Zu1DE0OzWjw7nnLZmJZYTDZZyEFHZdUhV8FkH5MCfoU1XMaxXovpyW5nq5scPqq0ZDP9Zyl04oQ==