Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Refactor .travis.yml to deduplicate uses of Docker and use Mustache comments #7351

Merged
merged 7 commits into from Mar 11, 2019
@@ -131,8 +131,6 @@ base_linux_test_config: &base_linux_test_config
before_install:
- PATH="/usr/lib/jvm/java-8-openjdk-amd64/jre/bin":$PATH
- JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64
# Increase the max number of user watches to ensure that watchman is able to watch all
# files in the working copy.
- sudo sysctl fs.inotify.max_user_watches=524288
- ./build-support/bin/install_aws_cli_for_ci.sh
before_script:
@@ -218,8 +216,6 @@ linux_with_fuse: &linux_with_fuse
before_install:
- PATH="/usr/lib/jvm/java-8-openjdk-amd64/jre/bin":$PATH
- JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64
# Increase the max number of user watches to ensure that watchman is able to watch all
# files in the working copy.
- sudo sysctl fs.inotify.max_user_watches=524288
- ./build-support/bin/install_aws_cli_for_ci.sh
- sudo apt-get install -y pkg-config fuse libfuse-dev
@@ -232,15 +228,6 @@ travis_docker_image: &travis_docker_image
- docker
before_script:
- ulimit -c unlimited
script:
- >
docker build
--rm -t ${docker_image_name}
--build-arg "TRAVIS_USER=$(id -un)"
--build-arg "TRAVIS_UID=$(id -u)"
--build-arg "TRAVIS_GROUP=$(id -gn)"
--build-arg "TRAVIS_GID=$(id -g)"
build-support/docker/${docker_image_name}/

# -------------------------------------------------------------------------
# Bootstrap engine shards
@@ -259,14 +246,13 @@ base_linux_build_engine: &base_linux_build_engine
--build-arg "TRAVIS_GROUP=$(id -gn)"
--build-arg "TRAVIS_GID=$(id -g)"
build-support/docker/${docker_image_name}/
# Note that:
# * We mount ${HOME} to cache the ${HOME}/.cache/pants/rust-toolchain.
# * We also build fs_util, to take advantage of the rust code built during bootstrapping.
- docker run --rm -t
- >
docker run
--rm -t
-v "${HOME}:/travis/home"
-v "${TRAVIS_BUILD_DIR}:/travis/workdir"
${docker_image_name}:latest
sh -c "./build-support/bin/ci.sh ${BOOTSTRAP_ARGS} && ./build-support/bin/release.sh -f"
sh -c "${docker_run_command}"
- aws --no-sign-request --region us-east-1 s3 cp ${TRAVIS_BUILD_DIR}/pants.pex ${BOOTSTRAPPED_PEX_URL_PREFIX}.${BOOTSTRAPPED_PEX_KEY_SUFFIX}

py27_linux_build_engine: &py27_linux_build_engine
@@ -275,6 +261,8 @@ py27_linux_build_engine: &py27_linux_build_engine
name: "Build Linux native engine and pants.pex (Py2.7 PEX)"
env:
- docker_image_name=travis_ci
# Note that we also build fs_util, to take advantage of the rust code built during bootstrapping.
- docker_run_command="./build-support/bin/ci.sh -2b && ./build-support/bin/release.sh -f"
# NB: Only the Py2.7 shard sets PREPARE_DEPLOY to cause the fs_util binary to be uploaded to S3:
# either linux shard could upload this binary, since it does not depend on python at all.
- PREPARE_DEPLOY=1
@@ -288,6 +276,8 @@ py36_linux_build_engine: &py36_linux_build_engine
name: "Build Linux native engine and pants.pex (Py3.6 PEX)"
env:
- docker_image_name=travis_ci
# Note that we also build fs_util, to take advantage of the rust code built during bootstrapping.
- docker_run_command="./build-support/bin/ci.sh -b && ./build-support/bin/release.sh -f"
- CACHE_NAME=linuxpexbuild.py36
- BOOTSTRAPPED_PEX_KEY_SUFFIX=py36.linux
- BOOTSTRAP_ARGS='-b'
@@ -409,16 +399,6 @@ py27_linux_build_wheels_no_ucs: &py27_linux_build_wheels_no_ucs
# compatibility. This is a Py2.7 shard, so it is not subject to #6985.
<<: *travis_docker_image
<<: *base_build_wheels

py27_linux_build_wheels_ucs2: &py27_linux_build_wheels_ucs2
<<: *py27_linux_config
<<: *py27_linux_build_wheels_no_ucs
<<: *native_engine_cache_config
name: "Build wheels - Linux and cp27m (UCS2)"
env:
- *base_build_wheels_env
- docker_image_name=travis_ci_py27_ucs2
- CACHE_NAME=linuxwheelsbuild.ucs2
script:
- >
docker build
@@ -428,13 +408,26 @@ py27_linux_build_wheels_ucs2: &py27_linux_build_wheels_ucs2
--build-arg "TRAVIS_GROUP=$(id -gn)"
--build-arg "TRAVIS_GID=$(id -g)"
build-support/docker/${docker_image_name}/
- docker run --rm -t
- >
docker run
--rm -t
-v "${HOME}:/travis/home"
-v "${TRAVIS_BUILD_DIR}:/travis/workdir"
${docker_image_name}:latest
sh -c "./build-support/bin/ci.sh -2b
&& ./build-support/bin/check_pants_pex_abi.py cp27m
&& RUN_PANTS_FROM_PEX=1 ./build-support/bin/release.sh -n"
sh -c "${docker_run_command}"
py27_linux_build_wheels_ucs2: &py27_linux_build_wheels_ucs2
<<: *py27_linux_config
<<: *py27_linux_build_wheels_no_ucs
<<: *native_engine_cache_config
name: "Build wheels - Linux and cp27m (UCS2)"
env:
- *base_build_wheels_env
- docker_image_name=travis_ci_py27_ucs2
- docker_run_command="./build-support/bin/ci.sh -2b
&& ./build-support/bin/check_pants_pex_abi.py cp27m
&& RUN_PANTS_FROM_PEX=1 ./build-support/bin/release.sh -n"
- CACHE_NAME=linuxwheelsbuild.ucs2

py27_linux_build_wheels_ucs4: &py27_linux_build_wheels_ucs4
<<: *py27_linux_build_wheels_no_ucs
@@ -446,22 +439,9 @@ py27_linux_build_wheels_ucs4: &py27_linux_build_wheels_ucs4
- *py27_linux_test_config_env
- *base_build_wheels_env
- docker_image_name=travis_ci
- docker_run_command="./build-support/bin/check_pants_pex_abi.py cp27mu
&& RUN_PANTS_FROM_PEX=1 ./build-support/bin/release.sh -n"
- CACHE_NAME=linuxwheelsbuild.ucs4
script:
- >
docker build
--rm -t ${docker_image_name}
--build-arg "TRAVIS_USER=$(id -un)"
--build-arg "TRAVIS_UID=$(id -u)"
--build-arg "TRAVIS_GROUP=$(id -gn)"
--build-arg "TRAVIS_GID=$(id -g)"
build-support/docker/${docker_image_name}/
- docker run --rm -t
-v "${HOME}:/travis/home"
-v "${TRAVIS_BUILD_DIR}:/travis/workdir"
${docker_image_name}:latest
sh -c "./build-support/bin/check_pants_pex_abi.py cp27mu
&& RUN_PANTS_FROM_PEX=1 ./build-support/bin/release.sh -n"

py27_osx_build_wheels_no_ucs: &py27_osx_build_wheels_no_ucs
<<: *base_build_wheels
@@ -1,6 +1,6 @@
- PATH="/usr/lib/jvm/java-8-openjdk-amd64/jre/bin":$PATH
- JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64
# Increase the max number of user watches to ensure that watchman is able to watch all
# files in the working copy.
{{! Increase the max number of user watches to ensure that watchman is able to watch all
# files in the working copy. }}
- sudo sysctl fs.inotify.max_user_watches=524288
- ./build-support/bin/install_aws_cli_for_ci.sh
@@ -0,0 +1,7 @@
{{! Note we mount ${HOME} to cache the ${HOME}/.cache/pants/rust-toolchain. }}
docker run
--rm -t
-v "${HOME}:/travis/home"
-v "${TRAVIS_BUILD_DIR}:/travis/workdir"
${docker_image_name}:latest
sh -c "${docker_run_command}"
@@ -1,5 +1,9 @@
{{! These flags are necessary to get OpenSSL working.
See https://github.com/pyenv/pyenv/wiki/Common-build-problems#error-the-python-ssl-extension-was-not-compiled-missing-the-openssl-lib. }}
PATH="/usr/local/opt/openssl/bin:$PATH"
LDFLAGS="-L/usr/local/opt/openssl/lib"
CPPFLAGS="-I/usr/local/opt/openssl/include"
{{! We directly will clone Pyenv as some older OSX image shards have too outdated of pyenv
to work as we need. So, we must explicitly add its entries to PATH. }}
This conversation was marked as resolved by Eric-Arellano

This comment has been minimized.

Copy link
@cosmicexplorer

cosmicexplorer Mar 10, 2019

Contributor

I might leave a TODO here or somewhere else pointing to #7064 where we plan to use the pyenv installer instead of cloning it directly (that PR will need an update now).

This comment has been minimized.

Copy link
@cosmicexplorer

cosmicexplorer Mar 10, 2019

Contributor

(actually, I am thinking we probably don't want to use pyenv-installer there, but the TODO might be cool)

PYENV_ROOT="${HOME}/.pyenv"
PATH="${PYENV_ROOT}/shims:${PATH}"
@@ -32,7 +32,8 @@ def get_mustache_file(file_name):
before_install_linux = get_mustache_file('before_install_linux.mustache')
before_install_osx = get_mustache_file('before_install_osx.mustache')
env_osx_with_pyenv = get_mustache_file('env_osx_with_pyenv.mustache')
launch_docker_image = get_mustache_file('launch_docker_image.mustache')
docker_build_image = get_mustache_file('docker_build_image.mustache')
docker_run_image = get_mustache_file('docker_run_image.mustache')

context = {
'header': HEADER,
@@ -47,6 +48,7 @@ def get_mustache_file(file_name):
'before_install_linux': before_install_linux,
'before_install_osx': before_install_osx,
'env_osx_with_pyenv': env_osx_with_pyenv,
'launch_docker_image': launch_docker_image
'docker_build_image': docker_build_image,
'docker_run_image': docker_run_image
})
print(renderer.render(template, context))
@@ -203,9 +203,6 @@ travis_docker_image: &travis_docker_image
- docker
before_script:
- ulimit -c unlimited
script:
- >
{{>launch_docker_image}}
# -------------------------------------------------------------------------
# Bootstrap engine shards
@@ -217,15 +214,9 @@ base_linux_build_engine: &base_linux_build_engine
stage: *bootstrap
script:
- >
{{>launch_docker_image}}
# Note that:
# * We mount ${HOME} to cache the ${HOME}/.cache/pants/rust-toolchain.
# * We also build fs_util, to take advantage of the rust code built during bootstrapping.
- docker run --rm -t
-v "${HOME}:/travis/home"
-v "${TRAVIS_BUILD_DIR}:/travis/workdir"
${docker_image_name}:latest
sh -c "./build-support/bin/ci.sh ${BOOTSTRAP_ARGS} && ./build-support/bin/release.sh -f"
{{>docker_build_image}}
- >
{{>docker_run_image}}
- aws --no-sign-request --region us-east-1 s3 cp ${TRAVIS_BUILD_DIR}/pants.pex ${BOOTSTRAPPED_PEX_URL_PREFIX}.${BOOTSTRAPPED_PEX_KEY_SUFFIX}
py27_linux_build_engine: &py27_linux_build_engine
@@ -234,6 +225,8 @@ py27_linux_build_engine: &py27_linux_build_engine
name: "Build Linux native engine and pants.pex (Py2.7 PEX)"
env:
- docker_image_name=travis_ci
# Note that we also build fs_util, to take advantage of the rust code built during bootstrapping.
- docker_run_command="./build-support/bin/ci.sh -2b && ./build-support/bin/release.sh -f"
# NB: Only the Py2.7 shard sets PREPARE_DEPLOY to cause the fs_util binary to be uploaded to S3:
# either linux shard could upload this binary, since it does not depend on python at all.
- PREPARE_DEPLOY=1
@@ -247,6 +240,8 @@ py36_linux_build_engine: &py36_linux_build_engine
name: "Build Linux native engine and pants.pex (Py3.6 PEX)"
env:
- docker_image_name=travis_ci
# Note that we also build fs_util, to take advantage of the rust code built during bootstrapping.
- docker_run_command="./build-support/bin/ci.sh -b && ./build-support/bin/release.sh -f"
- CACHE_NAME=linuxpexbuild.py36
- BOOTSTRAPPED_PEX_KEY_SUFFIX=py36.linux
- BOOTSTRAP_ARGS='-b'
@@ -368,6 +363,11 @@ py27_linux_build_wheels_no_ucs: &py27_linux_build_wheels_no_ucs
# compatibility. This is a Py2.7 shard, so it is not subject to #6985.
<<: *travis_docker_image
<<: *base_build_wheels
script:
- >
{{>docker_build_image}}
- >
{{>docker_run_image}}
py27_linux_build_wheels_ucs2: &py27_linux_build_wheels_ucs2
<<: *py27_linux_config
@@ -377,17 +377,10 @@ py27_linux_build_wheels_ucs2: &py27_linux_build_wheels_ucs2
env:
- *base_build_wheels_env
- docker_image_name=travis_ci_py27_ucs2
- docker_run_command="./build-support/bin/ci.sh -2b
&& ./build-support/bin/check_pants_pex_abi.py cp27m
&& RUN_PANTS_FROM_PEX=1 ./build-support/bin/release.sh -n"
This conversation was marked as resolved by Eric-Arellano

This comment has been minimized.

Copy link
@cosmicexplorer

cosmicexplorer Mar 10, 2019

Contributor

Having a comment somewhere on how this gets injected into the script that gets run would be neat, since there's a bit of indirection where ${docker_run_command} is used in the expansion of {{>docker_run_image}}.

- CACHE_NAME=linuxwheelsbuild.ucs2
script:
- >
{{>launch_docker_image}}
- docker run --rm -t
-v "${HOME}:/travis/home"
-v "${TRAVIS_BUILD_DIR}:/travis/workdir"
${docker_image_name}:latest
sh -c "./build-support/bin/ci.sh -2b
&& ./build-support/bin/check_pants_pex_abi.py cp27m
&& RUN_PANTS_FROM_PEX=1 ./build-support/bin/release.sh -n"
py27_linux_build_wheels_ucs4: &py27_linux_build_wheels_ucs4
<<: *py27_linux_build_wheels_no_ucs
@@ -399,16 +392,9 @@ py27_linux_build_wheels_ucs4: &py27_linux_build_wheels_ucs4
- *py27_linux_test_config_env
- *base_build_wheels_env
- docker_image_name=travis_ci
- docker_run_command="./build-support/bin/check_pants_pex_abi.py cp27mu
&& RUN_PANTS_FROM_PEX=1 ./build-support/bin/release.sh -n"
- CACHE_NAME=linuxwheelsbuild.ucs4
script:
- >
{{>launch_docker_image}}
- docker run --rm -t
-v "${HOME}:/travis/home"
-v "${TRAVIS_BUILD_DIR}:/travis/workdir"
${docker_image_name}:latest
sh -c "./build-support/bin/check_pants_pex_abi.py cp27mu
&& RUN_PANTS_FROM_PEX=1 ./build-support/bin/release.sh -n"
py27_osx_build_wheels_no_ucs: &py27_osx_build_wheels_no_ucs
<<: *base_build_wheels
ProTip! Use n and p to navigate between commits in a pull request.
You can’t perform that action at this time.