Enable dogfooding of remote caching via Toolchain in branch builds (#…

…11496) In a followup, we will turn this on for fork builds.
pantsbuild · Jan 25, 2021 · a2832b5 · a2832b5
1 parent 6f0ba91
commit a2832b5
Show file tree

Hide file tree

Showing 6 changed files with 117 additions and 37 deletions.
diff --git a/.travis.yml b/.travis.yml
@@ -54,6 +54,11 @@ jobs:
     before_install:
     - ./build-support/bin/install_aws_cli_for_ci.sh
     - pyenv global 2.7.17 3.6.10 3.7.6 3.8.1
+    - if [[ ${TRAVIS_PULL_REQUEST} == false ]]; then openssl aes-256-cbc -K $encrypted_f6717c01a353_key
+      -iv $encrypted_f6717c01a353_iv -in build-support/secrets/remote-cache-toolchain-jwt.txt.encrypted
+      -out build-support/secrets/remote-cache-toolchain-jwt.txt.decrypted -d && export
+      PANTS_REMOTE_OAUTH_BEARER_TOKEN_PATH=./build-support/secrets/remote-cache-toolchain-jwt.txt.decrypted;
+      fi
     cache:
       directories:
       - ${AWS_CLI_ROOT}
@@ -65,6 +70,7 @@ jobs:
       timeout: 500
     dist: bionic
     env:
+    - PANTS_REMOTE_CA_CERTS_PATH=/etc/ssl/certs/ca-certificates.crt
     - CACHE_NAME=bootstrap.linux.py37
     - BOOTSTRAPPED_PEX_KEY_SUFFIX=py37.linux
     language: python
@@ -109,6 +115,11 @@ jobs:
     before_install:
     - ./build-support/bin/install_aws_cli_for_ci.sh
     - pyenv global 2.7.17 3.6.10 3.7.6 3.8.1
+    - if [[ ${TRAVIS_PULL_REQUEST} == false ]]; then openssl aes-256-cbc -K $encrypted_f6717c01a353_key
+      -iv $encrypted_f6717c01a353_iv -in build-support/secrets/remote-cache-toolchain-jwt.txt.encrypted
+      -out build-support/secrets/remote-cache-toolchain-jwt.txt.decrypted -d && export
+      PANTS_REMOTE_OAUTH_BEARER_TOKEN_PATH=./build-support/secrets/remote-cache-toolchain-jwt.txt.decrypted;
+      fi
     cache:
       directories:
       - ${AWS_CLI_ROOT}
@@ -120,6 +131,7 @@ jobs:
       timeout: 500
     dist: bionic
     env:
+    - PANTS_REMOTE_CA_CERTS_PATH=/etc/ssl/certs/ca-certificates.crt
     - CACHE_NAME=bootstrap.linux.py38
     - BOOTSTRAPPED_PEX_KEY_SUFFIX=py38.linux
     language: python
@@ -254,9 +266,11 @@ jobs:
     - sudo sysctl fs.inotify.max_user_watches=524288
     - ./build-support/bin/install_aws_cli_for_ci.sh
     - pyenv global 2.7.17 3.6.10 3.7.6 3.8.1
-    - wget -qO- "https://github.com/crazy-max/travis-wait-enhanced/releases/download/v0.2.1/travis-wait-enhanced_0.2.1_linux_x86_64.tar.gz"
-      | tar -zxvf - travis-wait-enhanced
-    - mv travis-wait-enhanced /home/travis/bin/
+    - if [[ ${TRAVIS_PULL_REQUEST} == false ]]; then openssl aes-256-cbc -K $encrypted_f6717c01a353_key
+      -iv $encrypted_f6717c01a353_iv -in build-support/secrets/remote-cache-toolchain-jwt.txt.encrypted
+      -out build-support/secrets/remote-cache-toolchain-jwt.txt.decrypted -d && export
+      PANTS_REMOTE_OAUTH_BEARER_TOKEN_PATH=./build-support/secrets/remote-cache-toolchain-jwt.txt.decrypted;
+      fi
     before_script:
     - ./build-support/bin/get_ci_bootstrapped_pants_pex.sh ${AWS_BUCKET} ${BOOTSTRAPPED_PEX_KEY_PREFIX}.${BOOTSTRAPPED_PEX_KEY_SUFFIX}
     cache:
@@ -266,6 +280,7 @@ jobs:
       timeout: 500
     dist: bionic
     env:
+    - PANTS_REMOTE_CA_CERTS_PATH=/etc/ssl/certs/ca-certificates.crt
     - BOOTSTRAPPED_PEX_KEY_SUFFIX=py37.linux
     - CACHE_NAME=lint.py37
     language: python
@@ -279,8 +294,8 @@ jobs:
     - curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain
       none
     - source ${HOME}/.cargo/env
-    - travis-wait-enhanced --timeout 50m --interval 9m -- ./build-support/bin/ci.py
-      --githooks --smoke-tests --lint --python-version 3.7
+    - ./build-support/bin/ci.py --githooks --smoke-tests --lint --remote-cache-enabled
+      --python-version 3.7
     stage: Test Pants
   - addons:
       apt:
@@ -306,9 +321,11 @@ jobs:
     - sudo sysctl fs.inotify.max_user_watches=524288
     - ./build-support/bin/install_aws_cli_for_ci.sh
     - pyenv global 2.7.17 3.6.10 3.7.6 3.8.1
-    - wget -qO- "https://github.com/crazy-max/travis-wait-enhanced/releases/download/v0.2.1/travis-wait-enhanced_0.2.1_linux_x86_64.tar.gz"
-      | tar -zxvf - travis-wait-enhanced
-    - mv travis-wait-enhanced /home/travis/bin/
+    - if [[ ${TRAVIS_PULL_REQUEST} == false ]]; then openssl aes-256-cbc -K $encrypted_f6717c01a353_key
+      -iv $encrypted_f6717c01a353_iv -in build-support/secrets/remote-cache-toolchain-jwt.txt.encrypted
+      -out build-support/secrets/remote-cache-toolchain-jwt.txt.decrypted -d && export
+      PANTS_REMOTE_OAUTH_BEARER_TOKEN_PATH=./build-support/secrets/remote-cache-toolchain-jwt.txt.decrypted;
+      fi
     before_script:
     - ./build-support/bin/get_ci_bootstrapped_pants_pex.sh ${AWS_BUCKET} ${BOOTSTRAPPED_PEX_KEY_PREFIX}.${BOOTSTRAPPED_PEX_KEY_SUFFIX}
     cache:
@@ -318,6 +335,7 @@ jobs:
       timeout: 500
     dist: bionic
     env:
+    - PANTS_REMOTE_CA_CERTS_PATH=/etc/ssl/certs/ca-certificates.crt
     - BOOTSTRAPPED_PEX_KEY_SUFFIX=py38.linux
     - CACHE_NAME=lint.py38
     language: python
@@ -331,8 +349,8 @@ jobs:
     - curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain
       none
     - source ${HOME}/.cargo/env
-    - travis-wait-enhanced --timeout 50m --interval 9m -- ./build-support/bin/ci.py
-      --githooks --smoke-tests --lint --python-version 3.8
+    - ./build-support/bin/ci.py --githooks --smoke-tests --lint --remote-cache-enabled
+      --python-version 3.8
     stage: Test Pants (Cron)
   - before_cache:
     - sudo chown -R travis:travis "${HOME}" "${TRAVIS_BUILD_DIR}"
@@ -432,6 +450,11 @@ jobs:
     - wget -qO- "https://github.com/crazy-max/travis-wait-enhanced/releases/download/v0.2.1/travis-wait-enhanced_0.2.1_linux_x86_64.tar.gz"
       | tar -zxvf - travis-wait-enhanced
     - mv travis-wait-enhanced /home/travis/bin/
+    - if [[ ${TRAVIS_PULL_REQUEST} == false ]]; then openssl aes-256-cbc -K $encrypted_f6717c01a353_key
+      -iv $encrypted_f6717c01a353_iv -in build-support/secrets/remote-cache-toolchain-jwt.txt.encrypted
+      -out build-support/secrets/remote-cache-toolchain-jwt.txt.decrypted -d && export
+      PANTS_REMOTE_OAUTH_BEARER_TOKEN_PATH=./build-support/secrets/remote-cache-toolchain-jwt.txt.decrypted;
+      fi
     before_script:
     - ./build-support/bin/get_ci_bootstrapped_pants_pex.sh ${AWS_BUCKET} ${BOOTSTRAPPED_PEX_KEY_PREFIX}.${BOOTSTRAPPED_PEX_KEY_SUFFIX}
     cache:
@@ -441,6 +464,7 @@ jobs:
       timeout: 500
     dist: bionic
     env:
+    - PANTS_REMOTE_CA_CERTS_PATH=/etc/ssl/certs/ca-certificates.crt
     - BOOTSTRAPPED_PEX_KEY_SUFFIX=py37.linux
     - CACHE_NAME=python_tests.py37
     language: python
@@ -452,7 +476,7 @@ jobs:
     - '3.7'
     script:
     - travis-wait-enhanced --timeout 65m --interval 9m -- ./build-support/bin/ci.py
-      --unit-tests --integration-tests --python-version 3.7
+      --unit-tests --integration-tests --remote-cache-enabled --python-version 3.7
     stage: Test Pants
   - addons:
       apt:
@@ -483,6 +507,11 @@ jobs:
     - wget -qO- "https://github.com/crazy-max/travis-wait-enhanced/releases/download/v0.2.1/travis-wait-enhanced_0.2.1_linux_x86_64.tar.gz"
       | tar -zxvf - travis-wait-enhanced
     - mv travis-wait-enhanced /home/travis/bin/
+    - if [[ ${TRAVIS_PULL_REQUEST} == false ]]; then openssl aes-256-cbc -K $encrypted_f6717c01a353_key
+      -iv $encrypted_f6717c01a353_iv -in build-support/secrets/remote-cache-toolchain-jwt.txt.encrypted
+      -out build-support/secrets/remote-cache-toolchain-jwt.txt.decrypted -d && export
+      PANTS_REMOTE_OAUTH_BEARER_TOKEN_PATH=./build-support/secrets/remote-cache-toolchain-jwt.txt.decrypted;
+      fi
     before_script:
     - ./build-support/bin/get_ci_bootstrapped_pants_pex.sh ${AWS_BUCKET} ${BOOTSTRAPPED_PEX_KEY_PREFIX}.${BOOTSTRAPPED_PEX_KEY_SUFFIX}
     cache:
@@ -492,6 +521,7 @@ jobs:
       timeout: 500
     dist: bionic
     env:
+    - PANTS_REMOTE_CA_CERTS_PATH=/etc/ssl/certs/ca-certificates.crt
     - BOOTSTRAPPED_PEX_KEY_SUFFIX=py38.linux
     - CACHE_NAME=python_tests.py38
     language: python
@@ -503,7 +533,7 @@ jobs:
     - '3.7'
     script:
     - travis-wait-enhanced --timeout 65m --interval 9m -- ./build-support/bin/ci.py
-      --unit-tests --integration-tests --python-version 3.8
+      --unit-tests --integration-tests --remote-cache-enabled --python-version 3.8
     stage: Test Pants (Cron)
   - before_cache:
     - sudo chown -R travis:travis "${HOME}" "${TRAVIS_BUILD_DIR}"
@@ -602,6 +632,11 @@ jobs:
     - sudo sysctl fs.inotify.max_user_watches=524288
     - ./build-support/bin/install_aws_cli_for_ci.sh
     - pyenv global 2.7.17 3.6.10 3.7.6 3.8.1
+    - if [[ ${TRAVIS_PULL_REQUEST} == false ]]; then openssl aes-256-cbc -K $encrypted_f6717c01a353_key
+      -iv $encrypted_f6717c01a353_iv -in build-support/secrets/remote-cache-toolchain-jwt.txt.encrypted
+      -out build-support/secrets/remote-cache-toolchain-jwt.txt.decrypted -d && export
+      PANTS_REMOTE_OAUTH_BEARER_TOKEN_PATH=./build-support/secrets/remote-cache-toolchain-jwt.txt.decrypted;
+      fi
     before_script:
     - ./build-support/bin/get_ci_bootstrapped_pants_pex.sh ${AWS_BUCKET} ${BOOTSTRAPPED_PEX_KEY_PREFIX}.${BOOTSTRAPPED_PEX_KEY_SUFFIX}
     cache:
@@ -611,6 +646,7 @@ jobs:
       timeout: 500
     dist: bionic
     env:
+    - PANTS_REMOTE_CA_CERTS_PATH=/etc/ssl/certs/ca-certificates.crt
     - BOOTSTRAPPED_PEX_KEY_SUFFIX=py37.linux
     - PREPARE_DEPLOY=1
     - CACHE_NAME=wheels.linux
@@ -853,6 +889,11 @@ jobs:
     - sudo sysctl fs.inotify.max_user_watches=524288
     - ./build-support/bin/install_aws_cli_for_ci.sh
     - pyenv global 2.7.17 3.6.10 3.7.6 3.8.1
+    - if [[ ${TRAVIS_PULL_REQUEST} == false ]]; then openssl aes-256-cbc -K $encrypted_f6717c01a353_key
+      -iv $encrypted_f6717c01a353_iv -in build-support/secrets/remote-cache-toolchain-jwt.txt.encrypted
+      -out build-support/secrets/remote-cache-toolchain-jwt.txt.decrypted -d && export
+      PANTS_REMOTE_OAUTH_BEARER_TOKEN_PATH=./build-support/secrets/remote-cache-toolchain-jwt.txt.decrypted;
+      fi
     before_script:
     - ./build-support/bin/get_ci_bootstrapped_pants_pex.sh ${AWS_BUCKET} ${BOOTSTRAPPED_PEX_KEY_PREFIX}.${BOOTSTRAPPED_PEX_KEY_SUFFIX}
     cache:
@@ -872,6 +913,7 @@ jobs:
       skip_cleanup: true
     dist: bionic
     env:
+    - PANTS_REMOTE_CA_CERTS_PATH=/etc/ssl/certs/ca-certificates.crt
     - BOOTSTRAPPED_PEX_KEY_SUFFIX=py37.linux
     - RUN_PANTS_FROM_PEX=1
     - PANTS_PEX_RELEASE=stable
@@ -910,6 +952,11 @@ jobs:
     - sudo sysctl fs.inotify.max_user_watches=524288
     - ./build-support/bin/install_aws_cli_for_ci.sh
     - pyenv global 2.7.17 3.6.10 3.7.6 3.8.1
+    - if [[ ${TRAVIS_PULL_REQUEST} == false ]]; then openssl aes-256-cbc -K $encrypted_f6717c01a353_key
+      -iv $encrypted_f6717c01a353_iv -in build-support/secrets/remote-cache-toolchain-jwt.txt.encrypted
+      -out build-support/secrets/remote-cache-toolchain-jwt.txt.decrypted -d && export
+      PANTS_REMOTE_OAUTH_BEARER_TOKEN_PATH=./build-support/secrets/remote-cache-toolchain-jwt.txt.decrypted;
+      fi
     before_script:
     - ./build-support/bin/get_ci_bootstrapped_pants_pex.sh ${AWS_BUCKET} ${BOOTSTRAPPED_PEX_KEY_PREFIX}.${BOOTSTRAPPED_PEX_KEY_SUFFIX}
     cache:
@@ -919,6 +966,7 @@ jobs:
       timeout: 500
     dist: bionic
     env:
+    - PANTS_REMOTE_CA_CERTS_PATH=/etc/ssl/certs/ca-certificates.crt
     - BOOTSTRAPPED_PEX_KEY_SUFFIX=py37.linux
     - RUN_PANTS_FROM_PEX=1
     - PREPARE_DEPLOY=1

diff --git a/build-support/bin/ci.py b/build-support/bin/ci.py
@@ -32,7 +32,7 @@ def main() -> None:
     if args.smoke_tests:
         run_smoke_tests()
     if args.lint:
-        run_lint()
+        run_lint(remote_cache_enabled=args.remote_cache_enabled)
     if args.clippy:
         run_clippy()
     if args.cargo_audit:
@@ -41,6 +41,7 @@ def main() -> None:
         run_python_tests(
             include_unit=args.unit_tests,
             include_integration=args.integration_tests,
+            remote_cache_enabled=args.remote_cache_enabled,
         )
     if args.rust_tests:
         run_rust_tests()
@@ -74,6 +75,15 @@ def create_parser() -> argparse.ArgumentParser:
         default=PythonVersion.py37,
         help="Run Pants with this version.",
     )
+    parser.add_argument(
+        "--remote-cache-enabled",
+        action="store_true",
+        help=(
+            "Enable remote caching via Toolchain. This requires setting the options "
+            "`remote_oauth_bearer_token_path` and `remote_ca_certs_path` in your environment."
+        ),
+    )
+
     parser.add_argument(
         "--bootstrap", action="store_true", help="Bootstrap a pants.pex from local sources."
     )
@@ -91,6 +101,7 @@ def create_parser() -> argparse.ArgumentParser:
             "native_engine.so; this option should generally be avoided."
         ),
     )
+
     parser.add_argument("--githooks", action="store_true", help="Run pre-commit githook.")
     parser.add_argument(
         "--smoke-tests",
@@ -147,6 +158,9 @@ def set_run_from_pex() -> None:
     os.environ["RUN_PANTS_FROM_PEX"] = "1"
 
 
+IS_PR_BUILD = "CI" in os.environ and os.environ.get("TRAVIS_PULL_REQUEST", "false") != "false"
+
+
 # -------------------------------------------------------------------------
 # Bootstrap pants.pex
 # -------------------------------------------------------------------------
@@ -253,9 +267,11 @@ def run_check(command: List[str]) -> None:
             run_check(check)
 
 
-def run_lint() -> None:
+def run_lint(*, remote_cache_enabled: bool) -> None:
     targets = ["build-support::", "src::", "tests::"]
     command = ["./pants.pex", "--tag=-nolint", "lint", "typecheck", *targets]
+    if remote_cache_enabled and IS_PR_BUILD is False:
+        command.append("--pants-config-files=pants.remote-cache.toml")
     _run_command(
         command,
         slug="Lint",
@@ -306,17 +322,20 @@ def run_rust_tests() -> None:
             die("Rust test failure.")
 
 
-def run_python_tests(*, include_unit: bool, include_integration: bool) -> None:
-    if include_unit and include_integration:
-        extra_args = []
-    elif include_unit and not include_integration:
-        extra_args = ["--tag=-integration"]
-    elif not include_unit and include_integration:
-        extra_args = ["--tag=+integration"]
-    else:
+def run_python_tests(
+    *, include_unit: bool, include_integration: bool, remote_cache_enabled: bool
+) -> None:
+    extra_args = []
+    if remote_cache_enabled and IS_PR_BUILD is False:
+        extra_args.append("--pants-config-files=pants.remote-cache.toml")
+    if not include_unit and not include_integration:
         raise ValueError(
             "Must specify True for at least one of `include_unit` and `include_integration`."
         )
+    elif include_unit and not include_integration:
+        extra_args.append("--tag=-integration")
+    elif not include_unit and include_integration:
+        extra_args.append("--tag=+integration")
     _run_command(
         command=_test_command(extra_args=extra_args),
         slug="PythonTests",

diff --git a/build-support/bin/generate_travis_yml.py b/build-support/bin/generate_travis_yml.py
@@ -323,16 +323,25 @@ def linux_shard(
             }
         },
         "language": "python",
-        "before_install": _linux_before_install(
-            include_test_config=load_test_config, install_travis_wait=install_travis_wait
-        ),
+        "before_install": [
+            *_linux_before_install(
+                include_test_config=load_test_config, install_travis_wait=install_travis_wait
+            ),
+            (
+                "if [[ ${TRAVIS_PULL_REQUEST} == false ]]; then openssl aes-256-cbc -K "
+                "$encrypted_f6717c01a353_key -iv $encrypted_f6717c01a353_iv -in "
+                "build-support/secrets/remote-cache-toolchain-jwt.txt.encrypted -out "
+                "build-support/secrets/remote-cache-toolchain-jwt.txt.decrypted -d && export "
+                "PANTS_REMOTE_OAUTH_BEARER_TOKEN_PATH=./build-support/secrets/remote-cache-toolchain-jwt.txt.decrypted; fi"
+            ),
+        ],
         "after_failure": ["./build-support/bin/ci-failure.sh"],
         "stage": python_version.default_stage().value,
-        "env": [],
+        "env": ["PANTS_REMOTE_CA_CERTS_PATH=/etc/ssl/certs/ca-certificates.crt"],
     }
     if load_test_config:
         setup["before_script"] = [AWS_GET_PANTS_PEX_COMMAND]
-        setup["env"] = [f"BOOTSTRAPPED_PEX_KEY_SUFFIX=py{python_version.number}.linux"]
+        setup["env"].append(f"BOOTSTRAPPED_PEX_KEY_SUFFIX=py{python_version.number}.linux")
         setup = {**setup, **CACHE_PANTS_RUN}
     if use_docker:
         setup["services"] = ["docker"]
@@ -482,13 +491,13 @@ def bootstrap_osx(python_version: PythonVersion) -> Dict:
 
 def lint(python_version: PythonVersion) -> Dict:
     shard = {
-        **linux_shard(python_version=python_version, install_travis_wait=True),
+        **linux_shard(python_version=python_version),
         "name": f"Self-checks and lint (Python {python_version.decimal})",
         "script": [
             *_install_rust(),
             (
-                "travis-wait-enhanced --timeout 50m --interval 9m -- ./build-support/bin/ci.py "
-                f"--githooks --smoke-tests --lint --python-version {python_version.decimal}"
+                "./build-support/bin/ci.py --githooks --smoke-tests --lint "
+                f"--remote-cache-enabled --python-version {python_version.decimal}"
             ),
         ],
     }
@@ -536,8 +545,8 @@ def python_tests(python_version: PythonVersion) -> Dict:
         "name": f"Python tests (Python {python_version.decimal})",
         "script": [
             "travis-wait-enhanced --timeout 65m --interval 9m -- ./build-support/bin/ci.py "
-            "--unit-tests --integration-tests --python-version "
-            f"{python_version.decimal}"
+            "--unit-tests --integration-tests --remote-cache-enabled "
+            f"--python-version {python_version.decimal}"
         ],
         "after_success": ["./build-support/bin/upload_coverage.sh"],
     }

diff --git a/build-support/secrets/README.md b/build-support/secrets/README.md
@@ -0,0 +1,7 @@
+# Encrypted Secrets
+
+ This directory contains secrets encrypted via `travis encrypt-file`.
+
+(Travis has a limit on the size
+ of encrypted environment varibales. Thus, any secrets larger than that limit must be in files
+ encrypted in the repo.)
diff --git a/build-support/secrets/remote-cache-toolchain-jwt.txt.encrypted b/build-support/secrets/remote-cache-toolchain-jwt.txt.encrypted