Merge 9f4b770 into d771e60

pantsbuild · Dec 24, 2020 · 4549fbc · 4549fbc
2 parents d771e60 + 9f4b770
commit 4549fbc
Show file tree

Hide file tree

Showing 6 changed files with 86 additions and 9 deletions.
diff --git a/.travis.yml b/.travis.yml
@@ -54,6 +54,9 @@ jobs:
     before_install:
     - ./build-support/bin/install_aws_cli_for_ci.sh
     - pyenv global 2.7.17 3.6.10 3.7.6 3.8.1
+    - openssl aes-256-cbc -K $encrypted_f6717c01a353_key -iv $encrypted_f6717c01a353_iv
+      -in  build-support/secrets/remote-cache-toolchain-jwt.txt.encrypted  -out build-support/secrets/remote-cache-toolchain-jwt.txt.decrypted
+      -d
     cache:
       directories:
       - ${AWS_CLI_ROOT}
@@ -158,6 +161,9 @@ jobs:
     - wget -qO- "https://github.com/crazy-max/travis-wait-enhanced/releases/download/v0.2.1/travis-wait-enhanced_0.2.1_linux_x86_64.tar.gz"
       | tar -zxvf - travis-wait-enhanced
     - mv travis-wait-enhanced /home/travis/bin/
+    - openssl aes-256-cbc -K $encrypted_f6717c01a353_key -iv $encrypted_f6717c01a353_iv
+      -in  build-support/secrets/remote-cache-toolchain-jwt.txt.encrypted  -out build-support/secrets/remote-cache-toolchain-jwt.txt.decrypted
+      -d
     before_script:
     - ./build-support/bin/get_ci_bootstrapped_pants_pex.sh ${AWS_BUCKET} ${BOOTSTRAPPED_PEX_KEY_PREFIX}.${BOOTSTRAPPED_PEX_KEY_SUFFIX}
     cache:
@@ -190,6 +196,9 @@ jobs:
     - sudo sysctl fs.inotify.max_user_watches=524288
     - ./build-support/bin/install_aws_cli_for_ci.sh
     - pyenv global 2.7.15 3.6.7 3.7.1
+    - openssl aes-256-cbc -K $encrypted_f6717c01a353_key -iv $encrypted_f6717c01a353_iv
+      -in  build-support/secrets/remote-cache-toolchain-jwt.txt.encrypted  -out build-support/secrets/remote-cache-toolchain-jwt.txt.decrypted
+      -d
     - sudo apt-get install -y pkg-config fuse libfuse-dev
     - sudo modprobe fuse
     - sudo chmod 666 /dev/fuse
@@ -247,6 +256,9 @@ jobs:
     - wget -qO- "https://github.com/crazy-max/travis-wait-enhanced/releases/download/v0.2.1/travis-wait-enhanced_0.2.1_linux_x86_64.tar.gz"
       | tar -zxvf - travis-wait-enhanced
     - mv travis-wait-enhanced /home/travis/bin/
+    - openssl aes-256-cbc -K $encrypted_f6717c01a353_key -iv $encrypted_f6717c01a353_iv
+      -in  build-support/secrets/remote-cache-toolchain-jwt.txt.encrypted  -out build-support/secrets/remote-cache-toolchain-jwt.txt.decrypted
+      -d
     before_script:
     - ./build-support/bin/get_ci_bootstrapped_pants_pex.sh ${AWS_BUCKET} ${BOOTSTRAPPED_PEX_KEY_PREFIX}.${BOOTSTRAPPED_PEX_KEY_SUFFIX}
     cache:
@@ -267,7 +279,7 @@ jobs:
     - '3.7'
     script:
     - travis-wait-enhanced --timeout 65m --interval 9m -- ./build-support/bin/ci.py
-      --unit-tests --integration-tests --python-version 3.7
+      --unit-tests --integration-tests --remote-cache-enabled --python-version 3.7
     stage: Test Pants
   - before_cache:
     - sudo chown -R travis:travis "${HOME}" "${TRAVIS_BUILD_DIR}"
@@ -277,6 +289,9 @@ jobs:
     - sudo sysctl fs.inotify.max_user_watches=524288
     - ./build-support/bin/install_aws_cli_for_ci.sh
     - pyenv global 2.7.15 3.6.7 3.7.1
+    - openssl aes-256-cbc -K $encrypted_f6717c01a353_key -iv $encrypted_f6717c01a353_iv
+      -in  build-support/secrets/remote-cache-toolchain-jwt.txt.encrypted  -out build-support/secrets/remote-cache-toolchain-jwt.txt.decrypted
+      -d
     - sudo apt-get install -y pkg-config fuse libfuse-dev
     - sudo modprobe fuse
     - sudo chmod 666 /dev/fuse
@@ -366,6 +381,9 @@ jobs:
     - sudo sysctl fs.inotify.max_user_watches=524288
     - ./build-support/bin/install_aws_cli_for_ci.sh
     - pyenv global 2.7.17 3.6.10 3.7.6 3.8.1
+    - openssl aes-256-cbc -K $encrypted_f6717c01a353_key -iv $encrypted_f6717c01a353_iv
+      -in  build-support/secrets/remote-cache-toolchain-jwt.txt.encrypted  -out build-support/secrets/remote-cache-toolchain-jwt.txt.decrypted
+      -d
     before_script:
     - ./build-support/bin/get_ci_bootstrapped_pants_pex.sh ${AWS_BUCKET} ${BOOTSTRAPPED_PEX_KEY_PREFIX}.${BOOTSTRAPPED_PEX_KEY_SUFFIX}
     cache:
@@ -543,6 +561,9 @@ jobs:
     - sudo sysctl fs.inotify.max_user_watches=524288
     - ./build-support/bin/install_aws_cli_for_ci.sh
     - pyenv global 2.7.17 3.6.10 3.7.6 3.8.1
+    - openssl aes-256-cbc -K $encrypted_f6717c01a353_key -iv $encrypted_f6717c01a353_iv
+      -in  build-support/secrets/remote-cache-toolchain-jwt.txt.encrypted  -out build-support/secrets/remote-cache-toolchain-jwt.txt.decrypted
+      -d
     before_script:
     - ./build-support/bin/get_ci_bootstrapped_pants_pex.sh ${AWS_BUCKET} ${BOOTSTRAPPED_PEX_KEY_PREFIX}.${BOOTSTRAPPED_PEX_KEY_SUFFIX}
     cache:
@@ -600,6 +621,9 @@ jobs:
     - sudo sysctl fs.inotify.max_user_watches=524288
     - ./build-support/bin/install_aws_cli_for_ci.sh
     - pyenv global 2.7.17 3.6.10 3.7.6 3.8.1
+    - openssl aes-256-cbc -K $encrypted_f6717c01a353_key -iv $encrypted_f6717c01a353_iv
+      -in  build-support/secrets/remote-cache-toolchain-jwt.txt.encrypted  -out build-support/secrets/remote-cache-toolchain-jwt.txt.decrypted
+      -d
     before_script:
     - ./build-support/bin/get_ci_bootstrapped_pants_pex.sh ${AWS_BUCKET} ${BOOTSTRAPPED_PEX_KEY_PREFIX}.${BOOTSTRAPPED_PEX_KEY_SUFFIX}
     cache:

diff --git a/build-support/bin/ci.py b/build-support/bin/ci.py
@@ -21,9 +21,9 @@ def main() -> None:
     args = create_parser().parse_args()
     setup_environment(python_version=args.python_version)
 
-    with maybe_get_remote_execution_oauth_token_path(
-        remote_execution_enabled=args.remote_execution_enabled
-    ) as remote_execution_oauth_token_path:
+    with maybe_get_remote_cache_oauth_token_path(
+        remote_cache_enabled=args.remote_cache_enabled
+    ) as remote_cache_oauth_token_path:
 
         if args.bootstrap:
             bootstrap(
@@ -38,7 +38,7 @@ def main() -> None:
         if args.smoke_tests:
             run_smoke_tests()
         if args.lint:
-            run_lint(oauth_token_path=remote_execution_oauth_token_path)
+            run_lint(oauth_token_path=remote_cache_oauth_token_path)
         if args.clippy:
             run_clippy()
         if args.cargo_audit:
@@ -47,7 +47,7 @@ def main() -> None:
             run_python_tests(
                 include_unit=args.unit_tests,
                 include_integration=args.integration_tests,
-                oauth_token_path=remote_execution_oauth_token_path,
+                oauth_token_path=remote_cache_oauth_token_path,
             )
         if args.rust_tests:
             run_rust_tests()
@@ -88,6 +88,14 @@ def create_parser() -> argparse.ArgumentParser:
         "build execution permissions. If running in CI, the script will ping the secure token "
         "generator at https://github.com/pantsbuild/rbe-token-server.",
     )
+    parser.add_argument(
+        "--remote-cache-enabled",
+        action="store_true",
+        help="Pants will use the experimental remote cache service at build.toolchain.com:443 to cache "
+        "the results of processes using the cache features of Remote Execution API. Ths option "
+        "will only work when Pants is running with the Travis CI environment within the pantsbuild "
+        "organization given how the access token is encrypted.",
+    )
     parser.add_argument(
         "--bootstrap", action="store_true", help="Bootstrap a pants.pex from local sources."
     )
@@ -188,6 +196,16 @@ def maybe_get_remote_execution_oauth_token_path(
         yield tf.name
 
 
+@contextmanager
+def maybe_get_remote_cache_oauth_token_path(
+    *, remote_cache_enabled: bool
+) -> Iterator[Optional[str]]:
+    if not remote_cache_enabled:
+        yield None
+        return
+    yield "./build-support/secrets/remote-cache-toolchain-jwt.txt.decrypted"
+
+
 # -------------------------------------------------------------------------
 # Bootstrap pants.pex
 # -------------------------------------------------------------------------
@@ -242,6 +260,13 @@ def _use_remote_execution(oauth_token_path: str) -> List[str]:
     ]
 
 
+def _use_remote_cache(oauth_token_path: str) -> List[str]:
+    return [
+        "--pants-config-files=pants.remote-cache.toml",
+        f"--remote-oauth-bearer-token-path={oauth_token_path}",
+    ]
+
+
 def _run_command(
     command: List[str],
     *,
@@ -267,7 +292,7 @@ def _test_command(
     if extra_args:
         command.extend(extra_args)
     if oauth_token_path:
-        command.extend(_use_remote_execution(oauth_token_path))
+        command.extend(_use_remote_cache(oauth_token_path))
     return command
 
 
@@ -309,7 +334,7 @@ def run_lint(*, oauth_token_path: Optional[str] = None) -> None:
     targets = ["build-support::", "src::", "tests::"]
     command = ["./pants.pex", "--tag=-nolint", "lint", "typecheck", *targets]
     if oauth_token_path:
-        command.extend(_use_remote_execution(oauth_token_path))
+        command.extend(_use_remote_cache(oauth_token_path))
     _run_command(
         command,
         slug="Lint",

diff --git a/build-support/bin/generate_travis_yml.py b/build-support/bin/generate_travis_yml.py
@@ -288,6 +288,11 @@ def _linux_before_install(
                 "mv travis-wait-enhanced /home/travis/bin/",
             ]
         )
+    commands.append(
+        "openssl aes-256-cbc -K $encrypted_f6717c01a353_key -iv $encrypted_f6717c01a353_iv"
+        " -in  build-support/secrets/remote-cache-toolchain-jwt.txt.encrypted "
+        " -out build-support/secrets/remote-cache-toolchain-jwt.txt.decrypted -d"
+    )
     if include_test_config:
         return ["sudo sysctl fs.inotify.max_user_watches=524288", *commands]
     return commands
@@ -542,7 +547,7 @@ def python_tests(python_version: PythonVersion) -> Dict:
         "name": f"Python tests (Python {python_version.decimal})",
         "script": [
             "travis-wait-enhanced --timeout 65m --interval 9m -- ./build-support/bin/ci.py "
-            "--unit-tests --integration-tests --python-version "
+            "--unit-tests --integration-tests --remote-cache-enabled --python-version "
             f"{python_version.decimal}"
         ],
         "after_success": ["./build-support/bin/upload_coverage.sh"],

diff --git a/build-support/secrets/README.md b/build-support/secrets/README.md
@@ -0,0 +1,5 @@
+# Encrypted Secrets
+
+This directory contains secrets encrypted via `travis encrypt-file`. (Travis has a limit on the size
+of encrypted environment varibales. Thus, any secrets larger than that limit must be in files
+encrypted in the repo.)
diff --git a/build-support/secrets/remote-cache-toolchain-jwt.txt.encrypted b/build-support/secrets/remote-cache-toolchain-jwt.txt.encrypted
diff --git a/pants.remote-cache.toml b/pants.remote-cache.toml
@@ -0,0 +1,18 @@
+# Experimental remote cache configuration using Toolchain's remote cache service.
+
+[GLOBAL]
+remote_cache_read = true
+remote_cache_write = true
+
+remote_store_server = "build2.toolchain.com:443"
+remote_store_initial_timeout=250
+remote_store_timeout_multiplier=1.5
+remote_store_maximum_timeout=5000
+
+remote_oauth_bearer_token_path = "./build-support/secrets/remote-cache-toolchain-jwt.txt.decrypted"
+remote_instance_name = "main"
+
+# Enable SSL by pointing Pants at the default TLS certificate truststore on Linux.
+# Note: This will probably be different for macOS.
+remote_ca_certs_path = "/etc/ssl/certs/ca-certificates.crt"
+