Refactor .travis.yml to deduplicate uses of Docker and use Mustache comments

.travis.yml

@@ -131,8 +131,6 @@ base_linux_test_config: &base_linux_test_config
   before_install:
     - PATH="/usr/lib/jvm/java-8-openjdk-amd64/jre/bin":$PATH
     - JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64
-    # Increase the max number of user watches to ensure that watchman is able to watch all
-    # files in the working copy.
     - sudo sysctl fs.inotify.max_user_watches=524288
     - ./build-support/bin/install_aws_cli_for_ci.sh
   before_script:
@@ -218,8 +216,6 @@ linux_with_fuse: &linux_with_fuse
   before_install:
     - PATH="/usr/lib/jvm/java-8-openjdk-amd64/jre/bin":$PATH
     - JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64
-    # Increase the max number of user watches to ensure that watchman is able to watch all
-    # files in the working copy.
     - sudo sysctl fs.inotify.max_user_watches=524288
     - ./build-support/bin/install_aws_cli_for_ci.sh
     - sudo apt-get install -y pkg-config fuse libfuse-dev
@@ -232,15 +228,6 @@ travis_docker_image: &travis_docker_image
     - docker
   before_script:
     - ulimit -c unlimited
-  script:
-    - >
-      docker build
-      --rm -t ${docker_image_name}
-      --build-arg "TRAVIS_USER=$(id -un)"
-      --build-arg "TRAVIS_UID=$(id -u)"
-      --build-arg "TRAVIS_GROUP=$(id -gn)"
-      --build-arg "TRAVIS_GID=$(id -g)"
-      build-support/docker/${docker_image_name}/
 
 # -------------------------------------------------------------------------
 # Bootstrap engine shards
@@ -259,14 +246,13 @@ base_linux_build_engine: &base_linux_build_engine
       --build-arg "TRAVIS_GROUP=$(id -gn)"
       --build-arg "TRAVIS_GID=$(id -g)"
       build-support/docker/${docker_image_name}/
-    # Note that:
-    # * We mount ${HOME} to cache the ${HOME}/.cache/pants/rust-toolchain.
-    # * We also build fs_util, to take advantage of the rust code built during bootstrapping.
-    - docker run --rm -t
+    - >
+      docker run
+      --rm -t
       -v "${HOME}:/travis/home"
       -v "${TRAVIS_BUILD_DIR}:/travis/workdir"
       ${docker_image_name}:latest
-      sh -c "./build-support/bin/ci.sh ${BOOTSTRAP_ARGS} && ./build-support/bin/release.sh -f"
+      sh -c "${docker_run_command}"
     - aws --no-sign-request --region us-east-1 s3 cp ${TRAVIS_BUILD_DIR}/pants.pex ${BOOTSTRAPPED_PEX_URL_PREFIX}.${BOOTSTRAPPED_PEX_KEY_SUFFIX}
 
 py27_linux_build_engine: &py27_linux_build_engine
@@ -275,6 +261,8 @@ py27_linux_build_engine: &py27_linux_build_engine
   name: "Build Linux native engine and pants.pex (Py2.7 PEX)"
   env:
     - docker_image_name=travis_ci
+    # Note that we also build fs_util, to take advantage of the rust code built during bootstrapping.
+    - docker_run_command="./build-support/bin/ci.sh -2b && ./build-support/bin/release.sh -f"
     # NB: Only the Py2.7 shard sets PREPARE_DEPLOY to cause the fs_util binary to be uploaded to S3:
     # either linux shard could upload this binary, since it does not depend on python at all.
     - PREPARE_DEPLOY=1
@@ -288,6 +276,8 @@ py36_linux_build_engine: &py36_linux_build_engine
   name: "Build Linux native engine and pants.pex (Py3.6 PEX)"
   env:
     - docker_image_name=travis_ci
+    # Note that we also build fs_util, to take advantage of the rust code built during bootstrapping.
+    - docker_run_command="./build-support/bin/ci.sh -b && ./build-support/bin/release.sh -f"
     - CACHE_NAME=linuxpexbuild.py36
     - BOOTSTRAPPED_PEX_KEY_SUFFIX=py36.linux
     - BOOTSTRAP_ARGS='-b'
@@ -409,16 +399,6 @@ py27_linux_build_wheels_no_ucs: &py27_linux_build_wheels_no_ucs
   # compatibility. This is a Py2.7 shard, so it is not subject to #6985.
   <<: *travis_docker_image
   <<: *base_build_wheels
-
-py27_linux_build_wheels_ucs2: &py27_linux_build_wheels_ucs2
-  <<: *py27_linux_config
-  <<: *py27_linux_build_wheels_no_ucs
-  <<: *native_engine_cache_config
-  name: "Build wheels - Linux and cp27m (UCS2)"
-  env:
-    - *base_build_wheels_env
-    - docker_image_name=travis_ci_py27_ucs2
-    - CACHE_NAME=linuxwheelsbuild.ucs2
   script:
     - >
       docker build
@@ -428,13 +408,26 @@ py27_linux_build_wheels_ucs2: &py27_linux_build_wheels_ucs2
       --build-arg "TRAVIS_GROUP=$(id -gn)"
       --build-arg "TRAVIS_GID=$(id -g)"
       build-support/docker/${docker_image_name}/
-    - docker run --rm -t
+    - >
+      docker run
+      --rm -t
       -v "${HOME}:/travis/home"
       -v "${TRAVIS_BUILD_DIR}:/travis/workdir"
       ${docker_image_name}:latest
-      sh -c "./build-support/bin/ci.sh -2b
-             && ./build-support/bin/check_pants_pex_abi.py cp27m
-             && RUN_PANTS_FROM_PEX=1 ./build-support/bin/release.sh -n"
+      sh -c "${docker_run_command}"
+
+py27_linux_build_wheels_ucs2: &py27_linux_build_wheels_ucs2
+  <<: *py27_linux_config
+  <<: *py27_linux_build_wheels_no_ucs
+  <<: *native_engine_cache_config
+  name: "Build wheels - Linux and cp27m (UCS2)"
+  env:
+    - *base_build_wheels_env
+    - docker_image_name=travis_ci_py27_ucs2
+    - docker_run_command="./build-support/bin/ci.sh -2b
+                          && ./build-support/bin/check_pants_pex_abi.py cp27m
+                          && RUN_PANTS_FROM_PEX=1 ./build-support/bin/release.sh -n"
+    - CACHE_NAME=linuxwheelsbuild.ucs2
 
 py27_linux_build_wheels_ucs4: &py27_linux_build_wheels_ucs4
   <<: *py27_linux_build_wheels_no_ucs
@@ -446,22 +439,9 @@ py27_linux_build_wheels_ucs4: &py27_linux_build_wheels_ucs4
     - *py27_linux_test_config_env
     - *base_build_wheels_env
     - docker_image_name=travis_ci
+    - docker_run_command="./build-support/bin/check_pants_pex_abi.py cp27mu
+                          && RUN_PANTS_FROM_PEX=1 ./build-support/bin/release.sh -n"
     - CACHE_NAME=linuxwheelsbuild.ucs4
-  script:
-    - >
-      docker build
-      --rm -t ${docker_image_name}
-      --build-arg "TRAVIS_USER=$(id -un)"
-      --build-arg "TRAVIS_UID=$(id -u)"
-      --build-arg "TRAVIS_GROUP=$(id -gn)"
-      --build-arg "TRAVIS_GID=$(id -g)"
-      build-support/docker/${docker_image_name}/
-    - docker run --rm -t
-      -v "${HOME}:/travis/home"
-      -v "${TRAVIS_BUILD_DIR}:/travis/workdir"
-      ${docker_image_name}:latest
-      sh -c "./build-support/bin/check_pants_pex_abi.py cp27mu
-             && RUN_PANTS_FROM_PEX=1 ./build-support/bin/release.sh -n"
 
 py27_osx_build_wheels_no_ucs: &py27_osx_build_wheels_no_ucs
   <<: *base_build_wheels

build-support/travis/before_install_linux.mustache

@@ -1,6 +1,6 @@
 - PATH="/usr/lib/jvm/java-8-openjdk-amd64/jre/bin":$PATH
 - JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64
-# Increase the max number of user watches to ensure that watchman is able to watch all
-# files in the working copy.
+{{! Increase the max number of user watches to ensure that watchman is able to watch all
+# files in the working copy. }}
 - sudo sysctl fs.inotify.max_user_watches=524288
 - ./build-support/bin/install_aws_cli_for_ci.sh

build-support/travis/docker_run_image.mustache

@@ -0,0 +1,7 @@
+{{! Note we mount ${HOME} to cache the ${HOME}/.cache/pants/rust-toolchain. }}
+docker run
+--rm -t
+-v "${HOME}:/travis/home"
+-v "${TRAVIS_BUILD_DIR}:/travis/workdir"
+${docker_image_name}:latest
+sh -c "${docker_run_command}"

build-support/travis/env_osx_with_pyenv.mustache

@@ -1,5 +1,9 @@
+{{! These flags are necessary to get OpenSSL working.
+See https://github.com/pyenv/pyenv/wiki/Common-build-problems#error-the-python-ssl-extension-was-not-compiled-missing-the-openssl-lib. }}
 PATH="/usr/local/opt/openssl/bin:$PATH"
 LDFLAGS="-L/usr/local/opt/openssl/lib"
 CPPFLAGS="-I/usr/local/opt/openssl/include"
+{{! We directly will clone Pyenv as some older OSX image shards have too outdated of pyenv
+to work as we need. So, we must explicitly add its entries to PATH. }}
 PYENV_ROOT="${HOME}/.pyenv"
 PATH="${PYENV_ROOT}/shims:${PATH}"

build-support/travis/generate_travis_yml.py

@@ -32,7 +32,8 @@ def get_mustache_file(file_name):
   before_install_linux = get_mustache_file('before_install_linux.mustache')
   before_install_osx = get_mustache_file('before_install_osx.mustache')
   env_osx_with_pyenv = get_mustache_file('env_osx_with_pyenv.mustache')
-  launch_docker_image = get_mustache_file('launch_docker_image.mustache')
+  docker_build_image = get_mustache_file('docker_build_image.mustache')
+  docker_run_image = get_mustache_file('docker_run_image.mustache')
 
   context = {
     'header': HEADER,
@@ -47,6 +48,7 @@ def get_mustache_file(file_name):
     'before_install_linux': before_install_linux,
     'before_install_osx': before_install_osx,
     'env_osx_with_pyenv': env_osx_with_pyenv,
-    'launch_docker_image': launch_docker_image
+    'docker_build_image': docker_build_image,
+    'docker_run_image': docker_run_image
   })
   print(renderer.render(template, context))

build-support/travis/travis.yml.mustache

@@ -203,9 +203,6 @@ travis_docker_image: &travis_docker_image
     - docker
   before_script:
     - ulimit -c unlimited
-  script:
-    - >
-      {{>launch_docker_image}}
 
 # -------------------------------------------------------------------------
 # Bootstrap engine shards
@@ -217,15 +214,9 @@ base_linux_build_engine: &base_linux_build_engine
   stage: *bootstrap
   script:
     - >
-      {{>launch_docker_image}}
-    # Note that:
-    # * We mount ${HOME} to cache the ${HOME}/.cache/pants/rust-toolchain.
-    # * We also build fs_util, to take advantage of the rust code built during bootstrapping.
-    - docker run --rm -t
-      -v "${HOME}:/travis/home"
-      -v "${TRAVIS_BUILD_DIR}:/travis/workdir"
-      ${docker_image_name}:latest
-      sh -c "./build-support/bin/ci.sh ${BOOTSTRAP_ARGS} && ./build-support/bin/release.sh -f"
+      {{>docker_build_image}}
+    - >
+      {{>docker_run_image}}
     - aws --no-sign-request --region us-east-1 s3 cp ${TRAVIS_BUILD_DIR}/pants.pex ${BOOTSTRAPPED_PEX_URL_PREFIX}.${BOOTSTRAPPED_PEX_KEY_SUFFIX}
 
 py27_linux_build_engine: &py27_linux_build_engine
@@ -234,6 +225,8 @@ py27_linux_build_engine: &py27_linux_build_engine
   name: "Build Linux native engine and pants.pex (Py2.7 PEX)"
   env:
     - docker_image_name=travis_ci
+    # Note that we also build fs_util, to take advantage of the rust code built during bootstrapping.
+    - docker_run_command="./build-support/bin/ci.sh -2b && ./build-support/bin/release.sh -f"
     # NB: Only the Py2.7 shard sets PREPARE_DEPLOY to cause the fs_util binary to be uploaded to S3:
     # either linux shard could upload this binary, since it does not depend on python at all.
     - PREPARE_DEPLOY=1
@@ -247,6 +240,8 @@ py36_linux_build_engine: &py36_linux_build_engine
   name: "Build Linux native engine and pants.pex (Py3.6 PEX)"
   env:
     - docker_image_name=travis_ci
+    # Note that we also build fs_util, to take advantage of the rust code built during bootstrapping.
+    - docker_run_command="./build-support/bin/ci.sh -b && ./build-support/bin/release.sh -f"
     - CACHE_NAME=linuxpexbuild.py36
     - BOOTSTRAPPED_PEX_KEY_SUFFIX=py36.linux
     - BOOTSTRAP_ARGS='-b'
@@ -368,6 +363,11 @@ py27_linux_build_wheels_no_ucs: &py27_linux_build_wheels_no_ucs
   # compatibility. This is a Py2.7 shard, so it is not subject to #6985.
   <<: *travis_docker_image
   <<: *base_build_wheels
+  script:
+    - >
+      {{>docker_build_image}}
+    - >
+      {{>docker_run_image}}
 
 py27_linux_build_wheels_ucs2: &py27_linux_build_wheels_ucs2
   <<: *py27_linux_config
@@ -377,17 +377,10 @@ py27_linux_build_wheels_ucs2: &py27_linux_build_wheels_ucs2
   env:
     - *base_build_wheels_env
     - docker_image_name=travis_ci_py27_ucs2
+    - docker_run_command="./build-support/bin/ci.sh -2b
+                          && ./build-support/bin/check_pants_pex_abi.py cp27m
+                          && RUN_PANTS_FROM_PEX=1 ./build-support/bin/release.sh -n"
     - CACHE_NAME=linuxwheelsbuild.ucs2
-  script:
-    - >
-      {{>launch_docker_image}}
-    - docker run --rm -t
-      -v "${HOME}:/travis/home"
-      -v "${TRAVIS_BUILD_DIR}:/travis/workdir"
-      ${docker_image_name}:latest
-      sh -c "./build-support/bin/ci.sh -2b
-             && ./build-support/bin/check_pants_pex_abi.py cp27m
-             && RUN_PANTS_FROM_PEX=1 ./build-support/bin/release.sh -n"
 
 py27_linux_build_wheels_ucs4: &py27_linux_build_wheels_ucs4
   <<: *py27_linux_build_wheels_no_ucs
@@ -399,16 +392,9 @@ py27_linux_build_wheels_ucs4: &py27_linux_build_wheels_ucs4
     - *py27_linux_test_config_env
     - *base_build_wheels_env
     - docker_image_name=travis_ci
+    - docker_run_command="./build-support/bin/check_pants_pex_abi.py cp27mu
+                          && RUN_PANTS_FROM_PEX=1 ./build-support/bin/release.sh -n"
     - CACHE_NAME=linuxwheelsbuild.ucs4
-  script:
-    - >
-      {{>launch_docker_image}}
-    - docker run --rm -t
-      -v "${HOME}:/travis/home"
-      -v "${TRAVIS_BUILD_DIR}:/travis/workdir"
-      ${docker_image_name}:latest
-      sh -c "./build-support/bin/check_pants_pex_abi.py cp27mu
-             && RUN_PANTS_FROM_PEX=1 ./build-support/bin/release.sh -n"
 
 py27_osx_build_wheels_no_ucs: &py27_osx_build_wheels_no_ucs
   <<: *base_build_wheels