From a2832b5e620b86e77ad1818d858747e30bb69340 Mon Sep 17 00:00:00 2001 From: Eric Arellano <14852634+Eric-Arellano@users.noreply.github.com> Date: Mon, 25 Jan 2021 16:35:01 -0700 Subject: [PATCH] Enable dogfooding of remote caching via Toolchain in branch builds (#11496) In a followup, we will turn this on for fork builds. --- .travis.yml | 72 +++++++++++++++--- build-support/bin/ci.py | 39 +++++++--- build-support/bin/generate_travis_yml.py | 29 ++++--- build-support/secrets/README.md | 7 ++ .../remote-cache-toolchain-jwt.txt.encrypted | Bin 0 -> 256 bytes pants.remote-cache.toml | 7 +- 6 files changed, 117 insertions(+), 37 deletions(-) create mode 100644 build-support/secrets/README.md create mode 100644 build-support/secrets/remote-cache-toolchain-jwt.txt.encrypted diff --git a/.travis.yml b/.travis.yml index 894ffe8c5c9..4c880c80e28 100644 --- a/.travis.yml +++ b/.travis.yml @@ -54,6 +54,11 @@ jobs: before_install: - ./build-support/bin/install_aws_cli_for_ci.sh - pyenv global 2.7.17 3.6.10 3.7.6 3.8.1 + - if [[ ${TRAVIS_PULL_REQUEST} == false ]]; then openssl aes-256-cbc -K $encrypted_f6717c01a353_key + -iv $encrypted_f6717c01a353_iv -in build-support/secrets/remote-cache-toolchain-jwt.txt.encrypted + -out build-support/secrets/remote-cache-toolchain-jwt.txt.decrypted -d && export + PANTS_REMOTE_OAUTH_BEARER_TOKEN_PATH=./build-support/secrets/remote-cache-toolchain-jwt.txt.decrypted; + fi cache: directories: - ${AWS_CLI_ROOT} @@ -65,6 +70,7 @@ jobs: timeout: 500 dist: bionic env: + - PANTS_REMOTE_CA_CERTS_PATH=/etc/ssl/certs/ca-certificates.crt - CACHE_NAME=bootstrap.linux.py37 - BOOTSTRAPPED_PEX_KEY_SUFFIX=py37.linux language: python @@ -109,6 +115,11 @@ jobs: before_install: - ./build-support/bin/install_aws_cli_for_ci.sh - pyenv global 2.7.17 3.6.10 3.7.6 3.8.1 + - if [[ ${TRAVIS_PULL_REQUEST} == false ]]; then openssl aes-256-cbc -K $encrypted_f6717c01a353_key + -iv $encrypted_f6717c01a353_iv -in build-support/secrets/remote-cache-toolchain-jwt.txt.encrypted + -out build-support/secrets/remote-cache-toolchain-jwt.txt.decrypted -d && export + PANTS_REMOTE_OAUTH_BEARER_TOKEN_PATH=./build-support/secrets/remote-cache-toolchain-jwt.txt.decrypted; + fi cache: directories: - ${AWS_CLI_ROOT} @@ -120,6 +131,7 @@ jobs: timeout: 500 dist: bionic env: + - PANTS_REMOTE_CA_CERTS_PATH=/etc/ssl/certs/ca-certificates.crt - CACHE_NAME=bootstrap.linux.py38 - BOOTSTRAPPED_PEX_KEY_SUFFIX=py38.linux language: python @@ -254,9 +266,11 @@ jobs: - sudo sysctl fs.inotify.max_user_watches=524288 - ./build-support/bin/install_aws_cli_for_ci.sh - pyenv global 2.7.17 3.6.10 3.7.6 3.8.1 - - wget -qO- "https://github.com/crazy-max/travis-wait-enhanced/releases/download/v0.2.1/travis-wait-enhanced_0.2.1_linux_x86_64.tar.gz" - | tar -zxvf - travis-wait-enhanced - - mv travis-wait-enhanced /home/travis/bin/ + - if [[ ${TRAVIS_PULL_REQUEST} == false ]]; then openssl aes-256-cbc -K $encrypted_f6717c01a353_key + -iv $encrypted_f6717c01a353_iv -in build-support/secrets/remote-cache-toolchain-jwt.txt.encrypted + -out build-support/secrets/remote-cache-toolchain-jwt.txt.decrypted -d && export + PANTS_REMOTE_OAUTH_BEARER_TOKEN_PATH=./build-support/secrets/remote-cache-toolchain-jwt.txt.decrypted; + fi before_script: - ./build-support/bin/get_ci_bootstrapped_pants_pex.sh ${AWS_BUCKET} ${BOOTSTRAPPED_PEX_KEY_PREFIX}.${BOOTSTRAPPED_PEX_KEY_SUFFIX} cache: @@ -266,6 +280,7 @@ jobs: timeout: 500 dist: bionic env: + - PANTS_REMOTE_CA_CERTS_PATH=/etc/ssl/certs/ca-certificates.crt - BOOTSTRAPPED_PEX_KEY_SUFFIX=py37.linux - CACHE_NAME=lint.py37 language: python @@ -279,8 +294,8 @@ jobs: - curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain none - source ${HOME}/.cargo/env - - travis-wait-enhanced --timeout 50m --interval 9m -- ./build-support/bin/ci.py - --githooks --smoke-tests --lint --python-version 3.7 + - ./build-support/bin/ci.py --githooks --smoke-tests --lint --remote-cache-enabled + --python-version 3.7 stage: Test Pants - addons: apt: @@ -306,9 +321,11 @@ jobs: - sudo sysctl fs.inotify.max_user_watches=524288 - ./build-support/bin/install_aws_cli_for_ci.sh - pyenv global 2.7.17 3.6.10 3.7.6 3.8.1 - - wget -qO- "https://github.com/crazy-max/travis-wait-enhanced/releases/download/v0.2.1/travis-wait-enhanced_0.2.1_linux_x86_64.tar.gz" - | tar -zxvf - travis-wait-enhanced - - mv travis-wait-enhanced /home/travis/bin/ + - if [[ ${TRAVIS_PULL_REQUEST} == false ]]; then openssl aes-256-cbc -K $encrypted_f6717c01a353_key + -iv $encrypted_f6717c01a353_iv -in build-support/secrets/remote-cache-toolchain-jwt.txt.encrypted + -out build-support/secrets/remote-cache-toolchain-jwt.txt.decrypted -d && export + PANTS_REMOTE_OAUTH_BEARER_TOKEN_PATH=./build-support/secrets/remote-cache-toolchain-jwt.txt.decrypted; + fi before_script: - ./build-support/bin/get_ci_bootstrapped_pants_pex.sh ${AWS_BUCKET} ${BOOTSTRAPPED_PEX_KEY_PREFIX}.${BOOTSTRAPPED_PEX_KEY_SUFFIX} cache: @@ -318,6 +335,7 @@ jobs: timeout: 500 dist: bionic env: + - PANTS_REMOTE_CA_CERTS_PATH=/etc/ssl/certs/ca-certificates.crt - BOOTSTRAPPED_PEX_KEY_SUFFIX=py38.linux - CACHE_NAME=lint.py38 language: python @@ -331,8 +349,8 @@ jobs: - curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain none - source ${HOME}/.cargo/env - - travis-wait-enhanced --timeout 50m --interval 9m -- ./build-support/bin/ci.py - --githooks --smoke-tests --lint --python-version 3.8 + - ./build-support/bin/ci.py --githooks --smoke-tests --lint --remote-cache-enabled + --python-version 3.8 stage: Test Pants (Cron) - before_cache: - sudo chown -R travis:travis "${HOME}" "${TRAVIS_BUILD_DIR}" @@ -432,6 +450,11 @@ jobs: - wget -qO- "https://github.com/crazy-max/travis-wait-enhanced/releases/download/v0.2.1/travis-wait-enhanced_0.2.1_linux_x86_64.tar.gz" | tar -zxvf - travis-wait-enhanced - mv travis-wait-enhanced /home/travis/bin/ + - if [[ ${TRAVIS_PULL_REQUEST} == false ]]; then openssl aes-256-cbc -K $encrypted_f6717c01a353_key + -iv $encrypted_f6717c01a353_iv -in build-support/secrets/remote-cache-toolchain-jwt.txt.encrypted + -out build-support/secrets/remote-cache-toolchain-jwt.txt.decrypted -d && export + PANTS_REMOTE_OAUTH_BEARER_TOKEN_PATH=./build-support/secrets/remote-cache-toolchain-jwt.txt.decrypted; + fi before_script: - ./build-support/bin/get_ci_bootstrapped_pants_pex.sh ${AWS_BUCKET} ${BOOTSTRAPPED_PEX_KEY_PREFIX}.${BOOTSTRAPPED_PEX_KEY_SUFFIX} cache: @@ -441,6 +464,7 @@ jobs: timeout: 500 dist: bionic env: + - PANTS_REMOTE_CA_CERTS_PATH=/etc/ssl/certs/ca-certificates.crt - BOOTSTRAPPED_PEX_KEY_SUFFIX=py37.linux - CACHE_NAME=python_tests.py37 language: python @@ -452,7 +476,7 @@ jobs: - '3.7' script: - travis-wait-enhanced --timeout 65m --interval 9m -- ./build-support/bin/ci.py - --unit-tests --integration-tests --python-version 3.7 + --unit-tests --integration-tests --remote-cache-enabled --python-version 3.7 stage: Test Pants - addons: apt: @@ -483,6 +507,11 @@ jobs: - wget -qO- "https://github.com/crazy-max/travis-wait-enhanced/releases/download/v0.2.1/travis-wait-enhanced_0.2.1_linux_x86_64.tar.gz" | tar -zxvf - travis-wait-enhanced - mv travis-wait-enhanced /home/travis/bin/ + - if [[ ${TRAVIS_PULL_REQUEST} == false ]]; then openssl aes-256-cbc -K $encrypted_f6717c01a353_key + -iv $encrypted_f6717c01a353_iv -in build-support/secrets/remote-cache-toolchain-jwt.txt.encrypted + -out build-support/secrets/remote-cache-toolchain-jwt.txt.decrypted -d && export + PANTS_REMOTE_OAUTH_BEARER_TOKEN_PATH=./build-support/secrets/remote-cache-toolchain-jwt.txt.decrypted; + fi before_script: - ./build-support/bin/get_ci_bootstrapped_pants_pex.sh ${AWS_BUCKET} ${BOOTSTRAPPED_PEX_KEY_PREFIX}.${BOOTSTRAPPED_PEX_KEY_SUFFIX} cache: @@ -492,6 +521,7 @@ jobs: timeout: 500 dist: bionic env: + - PANTS_REMOTE_CA_CERTS_PATH=/etc/ssl/certs/ca-certificates.crt - BOOTSTRAPPED_PEX_KEY_SUFFIX=py38.linux - CACHE_NAME=python_tests.py38 language: python @@ -503,7 +533,7 @@ jobs: - '3.7' script: - travis-wait-enhanced --timeout 65m --interval 9m -- ./build-support/bin/ci.py - --unit-tests --integration-tests --python-version 3.8 + --unit-tests --integration-tests --remote-cache-enabled --python-version 3.8 stage: Test Pants (Cron) - before_cache: - sudo chown -R travis:travis "${HOME}" "${TRAVIS_BUILD_DIR}" @@ -602,6 +632,11 @@ jobs: - sudo sysctl fs.inotify.max_user_watches=524288 - ./build-support/bin/install_aws_cli_for_ci.sh - pyenv global 2.7.17 3.6.10 3.7.6 3.8.1 + - if [[ ${TRAVIS_PULL_REQUEST} == false ]]; then openssl aes-256-cbc -K $encrypted_f6717c01a353_key + -iv $encrypted_f6717c01a353_iv -in build-support/secrets/remote-cache-toolchain-jwt.txt.encrypted + -out build-support/secrets/remote-cache-toolchain-jwt.txt.decrypted -d && export + PANTS_REMOTE_OAUTH_BEARER_TOKEN_PATH=./build-support/secrets/remote-cache-toolchain-jwt.txt.decrypted; + fi before_script: - ./build-support/bin/get_ci_bootstrapped_pants_pex.sh ${AWS_BUCKET} ${BOOTSTRAPPED_PEX_KEY_PREFIX}.${BOOTSTRAPPED_PEX_KEY_SUFFIX} cache: @@ -611,6 +646,7 @@ jobs: timeout: 500 dist: bionic env: + - PANTS_REMOTE_CA_CERTS_PATH=/etc/ssl/certs/ca-certificates.crt - BOOTSTRAPPED_PEX_KEY_SUFFIX=py37.linux - PREPARE_DEPLOY=1 - CACHE_NAME=wheels.linux @@ -853,6 +889,11 @@ jobs: - sudo sysctl fs.inotify.max_user_watches=524288 - ./build-support/bin/install_aws_cli_for_ci.sh - pyenv global 2.7.17 3.6.10 3.7.6 3.8.1 + - if [[ ${TRAVIS_PULL_REQUEST} == false ]]; then openssl aes-256-cbc -K $encrypted_f6717c01a353_key + -iv $encrypted_f6717c01a353_iv -in build-support/secrets/remote-cache-toolchain-jwt.txt.encrypted + -out build-support/secrets/remote-cache-toolchain-jwt.txt.decrypted -d && export + PANTS_REMOTE_OAUTH_BEARER_TOKEN_PATH=./build-support/secrets/remote-cache-toolchain-jwt.txt.decrypted; + fi before_script: - ./build-support/bin/get_ci_bootstrapped_pants_pex.sh ${AWS_BUCKET} ${BOOTSTRAPPED_PEX_KEY_PREFIX}.${BOOTSTRAPPED_PEX_KEY_SUFFIX} cache: @@ -872,6 +913,7 @@ jobs: skip_cleanup: true dist: bionic env: + - PANTS_REMOTE_CA_CERTS_PATH=/etc/ssl/certs/ca-certificates.crt - BOOTSTRAPPED_PEX_KEY_SUFFIX=py37.linux - RUN_PANTS_FROM_PEX=1 - PANTS_PEX_RELEASE=stable @@ -910,6 +952,11 @@ jobs: - sudo sysctl fs.inotify.max_user_watches=524288 - ./build-support/bin/install_aws_cli_for_ci.sh - pyenv global 2.7.17 3.6.10 3.7.6 3.8.1 + - if [[ ${TRAVIS_PULL_REQUEST} == false ]]; then openssl aes-256-cbc -K $encrypted_f6717c01a353_key + -iv $encrypted_f6717c01a353_iv -in build-support/secrets/remote-cache-toolchain-jwt.txt.encrypted + -out build-support/secrets/remote-cache-toolchain-jwt.txt.decrypted -d && export + PANTS_REMOTE_OAUTH_BEARER_TOKEN_PATH=./build-support/secrets/remote-cache-toolchain-jwt.txt.decrypted; + fi before_script: - ./build-support/bin/get_ci_bootstrapped_pants_pex.sh ${AWS_BUCKET} ${BOOTSTRAPPED_PEX_KEY_PREFIX}.${BOOTSTRAPPED_PEX_KEY_SUFFIX} cache: @@ -919,6 +966,7 @@ jobs: timeout: 500 dist: bionic env: + - PANTS_REMOTE_CA_CERTS_PATH=/etc/ssl/certs/ca-certificates.crt - BOOTSTRAPPED_PEX_KEY_SUFFIX=py37.linux - RUN_PANTS_FROM_PEX=1 - PREPARE_DEPLOY=1 diff --git a/build-support/bin/ci.py b/build-support/bin/ci.py index 9bb4bfe9984..bd3878f003a 100755 --- a/build-support/bin/ci.py +++ b/build-support/bin/ci.py @@ -32,7 +32,7 @@ def main() -> None: if args.smoke_tests: run_smoke_tests() if args.lint: - run_lint() + run_lint(remote_cache_enabled=args.remote_cache_enabled) if args.clippy: run_clippy() if args.cargo_audit: @@ -41,6 +41,7 @@ def main() -> None: run_python_tests( include_unit=args.unit_tests, include_integration=args.integration_tests, + remote_cache_enabled=args.remote_cache_enabled, ) if args.rust_tests: run_rust_tests() @@ -74,6 +75,15 @@ def create_parser() -> argparse.ArgumentParser: default=PythonVersion.py37, help="Run Pants with this version.", ) + parser.add_argument( + "--remote-cache-enabled", + action="store_true", + help=( + "Enable remote caching via Toolchain. This requires setting the options " + "`remote_oauth_bearer_token_path` and `remote_ca_certs_path` in your environment." + ), + ) + parser.add_argument( "--bootstrap", action="store_true", help="Bootstrap a pants.pex from local sources." ) @@ -91,6 +101,7 @@ def create_parser() -> argparse.ArgumentParser: "native_engine.so; this option should generally be avoided." ), ) + parser.add_argument("--githooks", action="store_true", help="Run pre-commit githook.") parser.add_argument( "--smoke-tests", @@ -147,6 +158,9 @@ def set_run_from_pex() -> None: os.environ["RUN_PANTS_FROM_PEX"] = "1" +IS_PR_BUILD = "CI" in os.environ and os.environ.get("TRAVIS_PULL_REQUEST", "false") != "false" + + # ------------------------------------------------------------------------- # Bootstrap pants.pex # ------------------------------------------------------------------------- @@ -253,9 +267,11 @@ def run_check(command: List[str]) -> None: run_check(check) -def run_lint() -> None: +def run_lint(*, remote_cache_enabled: bool) -> None: targets = ["build-support::", "src::", "tests::"] command = ["./pants.pex", "--tag=-nolint", "lint", "typecheck", *targets] + if remote_cache_enabled and IS_PR_BUILD is False: + command.append("--pants-config-files=pants.remote-cache.toml") _run_command( command, slug="Lint", @@ -306,17 +322,20 @@ def run_rust_tests() -> None: die("Rust test failure.") -def run_python_tests(*, include_unit: bool, include_integration: bool) -> None: - if include_unit and include_integration: - extra_args = [] - elif include_unit and not include_integration: - extra_args = ["--tag=-integration"] - elif not include_unit and include_integration: - extra_args = ["--tag=+integration"] - else: +def run_python_tests( + *, include_unit: bool, include_integration: bool, remote_cache_enabled: bool +) -> None: + extra_args = [] + if remote_cache_enabled and IS_PR_BUILD is False: + extra_args.append("--pants-config-files=pants.remote-cache.toml") + if not include_unit and not include_integration: raise ValueError( "Must specify True for at least one of `include_unit` and `include_integration`." ) + elif include_unit and not include_integration: + extra_args.append("--tag=-integration") + elif not include_unit and include_integration: + extra_args.append("--tag=+integration") _run_command( command=_test_command(extra_args=extra_args), slug="PythonTests", diff --git a/build-support/bin/generate_travis_yml.py b/build-support/bin/generate_travis_yml.py index 64ea894594e..22431c8b3b6 100644 --- a/build-support/bin/generate_travis_yml.py +++ b/build-support/bin/generate_travis_yml.py @@ -323,16 +323,25 @@ def linux_shard( } }, "language": "python", - "before_install": _linux_before_install( - include_test_config=load_test_config, install_travis_wait=install_travis_wait - ), + "before_install": [ + *_linux_before_install( + include_test_config=load_test_config, install_travis_wait=install_travis_wait + ), + ( + "if [[ ${TRAVIS_PULL_REQUEST} == false ]]; then openssl aes-256-cbc -K " + "$encrypted_f6717c01a353_key -iv $encrypted_f6717c01a353_iv -in " + "build-support/secrets/remote-cache-toolchain-jwt.txt.encrypted -out " + "build-support/secrets/remote-cache-toolchain-jwt.txt.decrypted -d && export " + "PANTS_REMOTE_OAUTH_BEARER_TOKEN_PATH=./build-support/secrets/remote-cache-toolchain-jwt.txt.decrypted; fi" + ), + ], "after_failure": ["./build-support/bin/ci-failure.sh"], "stage": python_version.default_stage().value, - "env": [], + "env": ["PANTS_REMOTE_CA_CERTS_PATH=/etc/ssl/certs/ca-certificates.crt"], } if load_test_config: setup["before_script"] = [AWS_GET_PANTS_PEX_COMMAND] - setup["env"] = [f"BOOTSTRAPPED_PEX_KEY_SUFFIX=py{python_version.number}.linux"] + setup["env"].append(f"BOOTSTRAPPED_PEX_KEY_SUFFIX=py{python_version.number}.linux") setup = {**setup, **CACHE_PANTS_RUN} if use_docker: setup["services"] = ["docker"] @@ -482,13 +491,13 @@ def bootstrap_osx(python_version: PythonVersion) -> Dict: def lint(python_version: PythonVersion) -> Dict: shard = { - **linux_shard(python_version=python_version, install_travis_wait=True), + **linux_shard(python_version=python_version), "name": f"Self-checks and lint (Python {python_version.decimal})", "script": [ *_install_rust(), ( - "travis-wait-enhanced --timeout 50m --interval 9m -- ./build-support/bin/ci.py " - f"--githooks --smoke-tests --lint --python-version {python_version.decimal}" + "./build-support/bin/ci.py --githooks --smoke-tests --lint " + f"--remote-cache-enabled --python-version {python_version.decimal}" ), ], } @@ -536,8 +545,8 @@ def python_tests(python_version: PythonVersion) -> Dict: "name": f"Python tests (Python {python_version.decimal})", "script": [ "travis-wait-enhanced --timeout 65m --interval 9m -- ./build-support/bin/ci.py " - "--unit-tests --integration-tests --python-version " - f"{python_version.decimal}" + "--unit-tests --integration-tests --remote-cache-enabled " + f"--python-version {python_version.decimal}" ], "after_success": ["./build-support/bin/upload_coverage.sh"], } diff --git a/build-support/secrets/README.md b/build-support/secrets/README.md new file mode 100644 index 00000000000..8c7177ae98a --- /dev/null +++ b/build-support/secrets/README.md @@ -0,0 +1,7 @@ +# Encrypted Secrets + + This directory contains secrets encrypted via `travis encrypt-file`. + +(Travis has a limit on the size + of encrypted environment varibales. Thus, any secrets larger than that limit must be in files + encrypted in the repo.) \ No newline at end of file diff --git a/build-support/secrets/remote-cache-toolchain-jwt.txt.encrypted b/build-support/secrets/remote-cache-toolchain-jwt.txt.encrypted new file mode 100644 index 0000000000000000000000000000000000000000..3dc0148173facd28e741bd38c5a4e835d5412da4 GIT binary patch literal 256 zcmV+b0ssDYq!CVKylaEGSV-<8FOlA_2Va#47SK#~m>`3?RW7{U>o;d7DD^S7jMLQs zY>ztU zrUnDh`DQZx%DGzWplnWja z9@5JjwjXv5HZcah{08bJNYieOi_kSoH!%~K2?mI8S)x2PFLj5v)-}3RbpJ!^kFvl8 zHKg%4DmTX9_)`w6OdWR4oW0GMTcc%!anb$W*Zs+XBS_;Yz+FB9P`g67*(kD~B(NH9 G