From 12c12dba4aebaee3e56a70cc37e879b7c149803e Mon Sep 17 00:00:00 2001 From: Danny McClanahan <1305167+cosmicexplorer@users.noreply.github.com> Date: Mon, 14 Jan 2019 17:24:21 -0800 Subject: [PATCH] squash merge pants3-in-ci merge sha 2dfb39d80fb8a49570c9084ed8572eca0c06b0f5 and push to travis to ensure that PR works --- .travis.yml | 917 ++++++++++++------ build-support/bin/ci.sh | 43 +- build-support/bin/travis-ci.sh | 2 +- build-support/docker/travis_ci/Dockerfile | 21 +- build-support/travis/generate_travis_yml.py | 3 + build-support/travis/travis.yml.mustache | 644 ++++++++---- .../confluence/util/confluence_util.py | 2 +- .../tasks/go_binary_fingerprint_strategy.py | 2 +- ...ava_thrift_library_fingerprint_strategy.py | 2 +- 9 files changed, 1117 insertions(+), 519 deletions(-) diff --git a/.travis.yml b/.travis.yml index 2efe867b1282..d0bcd2eb96d0 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,12 +1,18 @@ - # GENERATED, DO NOT EDIT! # To change, edit build-support/travis/travis.yml.mustache and run # ./pants --quiet run build-support/travis:generate_travis_yml > .travis.yml +# +# Tip: Copy the generated `.travis.yml` into https://yamlvalidator.com to validate the YAML +# and see how the entries resolve to normalized JSON (helpful to debug anchors). # Conditions are documented here: https://docs.travis-ci.com/user/conditions-v1 conditions: v1 +# ------------------------------------------------------------------------- +# Global setup +# ------------------------------------------------------------------------- + env: global: - PANTS_CONFIG_FILES="${TRAVIS_BUILD_DIR}/pants.travis-ci.ini" @@ -28,6 +34,10 @@ stages: - name: &build_unstable Deploy Pants Pex Unstable if: tag IS NOT present AND type NOT IN (pull_request, cron) +# ------------------------------------------------------------------------- +# Cache config +# ------------------------------------------------------------------------- + # Travis cache config for jobs that build the native engine. native_engine_cache_config: &native_engine_cache_config before_cache: @@ -79,7 +89,11 @@ pants_run_cache_config: &pants_run_cache_config # https://github.com/pantsbuild/pants/issues/2485 - ${HOME}/.npm -default_linux_config: &default_linux_config +# ------------------------------------------------------------------------- +# Generic shard setups +# ------------------------------------------------------------------------- + +base_linux_config: &base_linux_config os: linux dist: xenial sudo: required @@ -96,16 +110,22 @@ default_linux_config: &default_linux_config - jq - unzip language: python - python: &python_version "2.7" before_install: - ./build-support/bin/install_aws_cli_for_ci.sh after_failure: - ./build-support/bin/ci-failure.sh -default_linux_test_config: &default_linux_test_config - <<: *default_linux_config +py2_linux_config: &py2_linux_config + <<: *base_linux_config + python: &python_version "2.7" + +py3_linux_config: &py3_linux_config + <<: *base_linux_config + python: &python_version "3.6" + +base_linux_test_config: &base_linux_test_config + <<: *base_linux_config <<: *pants_run_cache_config - stage: *test before_install: - PATH="/usr/lib/jvm/java-8-openjdk-amd64/jre/bin":$PATH - JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64 @@ -114,9 +134,23 @@ default_linux_test_config: &default_linux_test_config - sudo sysctl fs.inotify.max_user_watches=524288 - ./build-support/bin/install_aws_cli_for_ci.sh before_script: - - ./build-support/bin/get_ci_bootstrapped_pants_pex.sh ${BOOTSTRAPPED_PEX_BUCKET} ${BOOTSTRAPPED_PEX_KEY_PREFIX}.linux + - ./build-support/bin/get_ci_bootstrapped_pants_pex.sh ${BOOTSTRAPPED_PEX_BUCKET} ${BOOTSTRAPPED_PEX_KEY_PREFIX}.${BOOTSTRAPPED_PEX_KEY_SUFFIX} + +py2_linux_test_config: &py2_linux_test_config + <<: *py2_linux_config + <<: *base_linux_test_config + stage: *cron + env: + - &py2_linux_test_config_env BOOTSTRAPPED_PEX_KEY_SUFFIX=py2.linux + +py3_linux_test_config: &py3_linux_test_config + <<: *py3_linux_config + <<: *base_linux_test_config + stage: *test + env: + - &py3_linux_test_config_env BOOTSTRAPPED_PEX_KEY_SUFFIX=py3.linux -default_osx_config: &default_osx_config +base_osx_config: &base_osx_config os: osx language: generic before_install: @@ -124,14 +158,48 @@ default_osx_config: &default_osx_config - chmod 755 /usr/local/bin/jq - ./build-support/bin/install_aws_cli_for_ci.sh -default_osx_test_config: &default_osx_test_config - <<: *default_osx_config +py2_osx_config: &py2_osx_config + <<: *base_osx_config + +py3_osx_config: &py3_osx_config + <<: *base_osx_config + addons: + brew: + packages: &py3_osx_config_brew_packages + - pyenv + - openssl + env: + # Fix Python 3 issue linking to OpenSSL + - PATH="/usr/local/opt/openssl/bin:$PATH" LDFLAGS="-L/usr/local/opt/openssl/lib" CPPFLAGS="-I/usr/local/opt/openssl/include" + before_install: + - curl -L https://github.com/stedolan/jq/releases/download/jq-1.5/jq-osx-amd64 -o /usr/local/bin/jq + - chmod 755 /usr/local/bin/jq + - ./build-support/bin/install_aws_cli_for_ci.sh + # Install pyenv the conventional way, instead of through addon, due to execution order + - brew install pyenv + - pyenv install 3.6.8 && pyenv global 3.6.8 + +base_osx_test_config: &base_osx_test_config <<: *pants_run_cache_config - stage: *test before_script: - ulimit -c unlimited - ulimit -n 8192 - - ./build-support/bin/get_ci_bootstrapped_pants_pex.sh ${BOOTSTRAPPED_PEX_BUCKET} ${BOOTSTRAPPED_PEX_KEY_PREFIX}.osx + - ./build-support/bin/get_ci_bootstrapped_pants_pex.sh ${BOOTSTRAPPED_PEX_BUCKET} ${BOOTSTRAPPED_PEX_KEY_PREFIX}.${BOOTSTRAPPED_PEX_KEY_SUFFIX} + +py2_osx_test_config: &py2_osx_test_config + <<: *py2_osx_config + <<: *base_osx_test_config + stage: *cron + env: + - &py2_osx_test_config_env BOOTSTRAPPED_PEX_KEY_SUFFIX=py2.osx + +py3_osx_test_config: &py3_osx_test_config + <<: *py3_osx_config + <<: *base_osx_test_config + stage: *test + env: + # Must duplicate OpenSSL config from py3_osx_config's env because it cannot be merged into a new anchor + - &py3_osx_test_config_env PATH="/usr/local/opt/openssl/bin:$PATH" LDFLAGS="-L/usr/local/opt/openssl/lib" CPPFLAGS="-I/usr/local/opt/openssl/include" BOOTSTRAPPED_PEX_KEY_SUFFIX=py3.osx linux_with_fuse: &linux_with_fuse before_install: @@ -146,506 +214,746 @@ linux_with_fuse: &linux_with_fuse - sudo chmod 666 /dev/fuse - sudo chown root:$USER /etc/fuse.conf +# ------------------------------------------------------------------------- +# Bootstrap engine shards +# ------------------------------------------------------------------------- + +base_linux_build_engine: &base_linux_build_engine + <<: *native_engine_cache_config + stage: *bootstrap + services: + - docker + env: + - &base_linux_build_engine_env PREPARE_DEPLOY=1 # To deploy fs_util. + before_script: + - ulimit -c unlimited + script: + - docker build --rm -t travis_ci + --build-arg "TRAVIS_USER=$(id -un)" + --build-arg "TRAVIS_UID=$(id -u)" + --build-arg "TRAVIS_GROUP=$(id -gn)" + --build-arg "TRAVIS_GID=$(id -g)" + build-support/docker/travis_ci/ + # Note that: + # * We mount ${HOME} to cache the ${HOME}/.cache/pants/rust-toolchain. + # * With no args ci.sh just bootstraps a pants.pex. + # * We also build fs_util, to take advantage of the rust code built during bootstrapping. + - docker run --rm -t + -v "${HOME}:/travis/home" + -v "${TRAVIS_BUILD_DIR}:/travis/workdir" + travis_ci:latest + sh -c "source ~/.bashrc && ./build-support/bin/ci.sh ${BOOTSTRAP_ARGS} && ./build-support/bin/release.sh -f" + - aws --no-sign-request --region us-east-1 s3 cp ${TRAVIS_BUILD_DIR}/pants.pex ${BOOTSTRAPPED_PEX_URL_PREFIX}.${BOOTSTRAPPED_PEX_KEY_SUFFIX} + +py2_linux_build_engine: &py2_linux_build_engine + <<: *py2_linux_config + <<: *base_linux_build_engine + name: "Build Linux native engine and pants.pex (Py2 PEX)" + env: + - *base_linux_build_engine_env + - CACHE_NAME=linuxpexbuild.py2 + - BOOTSTRAPPED_PEX_KEY_SUFFIX=py2.linux + - BOOTSTRAP_ARGS='-2' + +py3_linux_build_engine: &py3_linux_build_engine + <<: *py3_linux_config + <<: *base_linux_build_engine + name: "Build Linux native engine and pants.pex (Py3 PEX)" + env: + - *base_linux_build_engine_env + - CACHE_NAME=linuxpexbuild.py3 + - BOOTSTRAPPED_PEX_KEY_SUFFIX=py3.linux + - BOOTSTRAP_ARGS='' + +base_osx_build_engine: &base_osx_build_engine + <<: *native_engine_cache_config + stage: *bootstrap + # We request the oldest image we can (corresponding to OSX 10.11) for maximum compatibility. + # We use 10.11 as a minimum to avoid https://github.com/rust-lang/regex/issues/489. + # See: https://docs.travis-ci.com/user/reference/osx/#OS-X-Version + osx_image: xcode8 + env: + - &base_osx_build_engine_env PREPARE_DEPLOY=1 # To deploy fs_util. + script: + # With no args ci.sh just bootstraps a pants.pex. We also build fs_util, to take advantage + # of the rust code built during bootstrapping. + - ./build-support/bin/ci.sh ${BOOTSTRAP_ARGS} && ./build-support/bin/release.sh -f + - aws --no-sign-request --region us-east-1 s3 cp ${TRAVIS_BUILD_DIR}/pants.pex ${BOOTSTRAPPED_PEX_URL_PREFIX}.${BOOTSTRAPPED_PEX_KEY_SUFFIX} + after_failure: + - ./build-support/bin/ci-failure.sh + +py2_osx_build_engine: &py2_osx_build_engine + <<: *py2_osx_config + <<: *base_osx_build_engine + name: "Build OSX native engine and pants.pex (Py2 PEX)" + env: + - *base_osx_build_engine_env + - CACHE_NAME=osxpexbuild.py2 + - BOOTSTRAPPED_PEX_KEY_SUFFIX=py2.osx + - BOOTSTRAP_ARGS='-2' + +py3_osx_build_engine: &py3_osx_build_engine + <<: *py3_osx_config + <<: *base_osx_build_engine + name: "Build OSX native engine and pants.pex (Py3 PEX)" + + env: + - *base_osx_build_engine_env + - CACHE_NAME=osxpexbuild.py3 + - BOOTSTRAPPED_PEX_KEY_SUFFIX=py3.osx + - BOOTSTRAP_ARGS='' + +# ------------------------------------------------------------------------- +# Build wheels +# ------------------------------------------------------------------------- + +base_build_wheels: &base_build_wheels + env: + - &base_build_wheels_env RUN_PANTS_FROM_PEX=1 PREPARE_DEPLOY=1 + script: + - ./build-support/bin/release.sh -n + +py2_linux_build_wheels: &py2_linux_build_wheels + <<: *py2_linux_test_config + <<: *base_build_wheels + stage: *cron + name: "Build Linux wheels (Py2 PEX)" + env: + - *py2_linux_test_config_env + - *base_build_wheels_env + - CACHE_NAME=linuxwheelsbuild.py2 + +py3_linux_build_wheels: &py3_linux_build_wheels + <<: *py3_linux_test_config + <<: *base_build_wheels + name: "Build Linux wheels (Py3 PEX)" + env: + - *py3_linux_test_config_env + - *base_build_wheels_env + - CACHE_NAME=linuxwheelsbuild.py3 + +py2_osx_build_wheels: &py2_osx_build_wheels + <<: *py2_osx_test_config + <<: *base_build_wheels + name: "Build OSX wheels (Py2 PEX)" + stage: *cron + env: + - *py2_osx_test_config_env + - *base_build_wheels_env + - CACHE_NAME=osxwheelsbuild.py2 + +py3_osx_build_wheels: &py3_osx_build_wheels + <<: *py3_osx_test_config + <<: *base_build_wheels + name: "Build OSX wheels (Py3 PEX)" + env: + - *py3_osx_test_config_env + - *base_build_wheels_env + - CACHE_NAME=osxwheelsbuild.py3 + +# ------------------------------------------------------------------------- +# OSX sanity checks +# ------------------------------------------------------------------------- + +base_osx_sanity_check: &base_osx_sanity_check + script: + - MODE=debug ./build-support/bin/travis-ci.sh -bm + +# TODO: Update this to use 10.14 once it is available +base_osx_10_12_sanity_check: &base_osx_10_12_sanity_check + <<: *base_osx_sanity_check + osx_image: xcode9.2 + +py2_osx_10_12_sanity_check: &py2_osx_10_12_sanity_check + <<: *py2_osx_test_config + <<: *base_osx_10_12_sanity_check + name: "OSX 10.12 sanity check (Py2 PEX)" + stage: *cron + env: + - *py2_osx_test_config_env + - CACHE_NAME=macos10.12sanity.py2 + +py3_osx_10_12_sanity_check: &py3_osx_10_12_sanity_check + <<: *py3_osx_test_config + <<: *base_osx_10_12_sanity_check + name: "OSX 10.12 sanity check (Py3 PEX)" + env: + - *py3_osx_test_config_env + - CACHE_NAME=macos10.12sanity.py3 + +base_osx_10_13_sanity_check: &base_osx_10_13_sanity_check + <<: *base_osx_sanity_check + osx_image: xcode10.1 + +py2_osx_10_13_sanity_check: &py2_osx_10_13_sanity_check + <<: *py2_osx_test_config + <<: *base_osx_10_13_sanity_check + name: "OSX 10.13 sanity check (Py2 PEX)" + stage: *cron + env: + - *py2_osx_test_config_env + - CACHE_NAME=macos10.13sanity.py2 + +py3_osx_10_13_sanity_check: &py3_osx_10_13_sanity_check + <<: *py3_osx_test_config + <<: *base_osx_10_13_sanity_check + name: "OSX 10.13 sanity check (Py3 PEX)" + env: + - *py3_osx_test_config_env + - CACHE_NAME=macos10.13sanity.py3 + +# ------------------------------------------------------------------------- +# Lint +# ------------------------------------------------------------------------- + +base_lint: &base_lint + <<: *linux_with_fuse + script: + - ./build-support/bin/travis-ci.sh -fbmrjt + +py2_lint: &py2_lint + <<: *py2_linux_test_config + <<: *base_lint + name: "Self-checks, lint, and JVM tests (Py2 PEX)" + env: + - *py2_linux_test_config_env + - CACHE_NAME=linuxselfchecks.py2 + +py3_lint: &py3_lint + <<: *py3_linux_test_config + <<: *base_lint + name: "Self-checks, lint, and JVM tests (Py3 PEX)" + env: + - *py3_linux_test_config_env + - CACHE_NAME=linuxselfchecks.py3 + +# ------------------------------------------------------------------------- +# Deploy +# ------------------------------------------------------------------------- + +deploy_stable_multiplatform_pex: &deploy_stable_multiplatform_pex + name: "Deploy stable multiplatform pants.pex" + os: linux + language: python + stage: *build_stable + env: + - CACHE_NAME=linuxpexdeploystable + - RUN_PANTS_FROM_PEX=1 + - PANTS_PEX_RELEASE=stable + script: + - ./build-support/bin/release.sh -p + deploy: + # See https://docs.travis-ci.com/user/deployment/releases/ + provider: releases + # The pantsbuild-ci-bot OAuth token, see the pantsbuild vault for details. + api_key: + secure: "u0aCsiuVGOg28YxG0sQUovuUm29kKwQfFgHbNz2TT5L+cGoHxGl4aoVOCtuwWYEtbNGmYc8/3WRS3C/jOiqQj6JEgHUzWOsnfKUObEqNhisAmXbzBbKc0wPQTL8WNK+DKFh32sD3yPYcw+a5PTLO56+o7rqlI25LK7A17WesHC4=" + file_glob: true + file: dist/deploy/pex/* + skip_cleanup: true + on: + # We only release a pex for Pants releases, which are tagged. + tags: true + repo: pantsbuild/pants + +deploy_unstable_multiplatform_pex: &deploy_unstable_multiplatform_pex + name: "Deploy unstable multiplatform pants.pex" + os: linux + language: python + stage: *build_unstable + env: + - CACHE_NAME=linuxpexdeployunstable + - RUN_PANTS_FROM_PEX=1 + - PREPARE_DEPLOY=1 + script: + - ./build-support/bin/release.sh -p && mkdir -p dist/deploy/pex/ && mv dist/pants*.pex dist/deploy/pex/ + +# ------------------------------------------------------------------------- +# Rust +# ------------------------------------------------------------------------- + +base_linux_rust_tests: &base_linux_rust_tests + <<: *linux_with_fuse + <<: *native_engine_cache_config + env: + - CACHE_NAME=linuxrusttests + os: linux + dist: xenial + sudo: required + language: python + before_script: + - ulimit -c unlimited + - ulimit -n 8192 + +py2_linux_rust_tests: &py2_linux_rust_tests + <<: *base_linux_rust_tests + name: "Linux Rust tests (Py2 PEX)" + python: 2.7 + stage: *cron + env: + - *py2_linux_test_config_env + - CACHE_NAME=linuxrusttests.py2 + script: + - ./build-support/bin/travis-ci.sh -be2 + +py3_linux_rust_tests: &py3_linux_rust_tests + <<: *base_linux_rust_tests + name: "Linux Rust tests (Py3 PEX)" + python: 3.6 + stage: *test + env: + - *py3_linux_test_config_env + - CACHE_NAME=linuxrusttests.py3 + script: + - ./build-support/bin/travis-ci.sh -be + +base_osx_rust_tests: &base_osx_rust_tests + # Fuse actually works on this image. It hangs on many others. + osx_image: xcode8.3 + addons: + homebrew: + casks: &base_osx_rust_tests_brew_casks + - osxfuse + +py2_osx_rust_tests: &py2_osx_rust_tests + <<: *base_osx_rust_tests + <<: *py2_osx_test_config + name: "OSX Rust + platform-specific tests (Py2 PEX)" + env: + - *py2_osx_test_config_env + - CACHE_NAME=macosrusttests.py2 + script: + - ./build-support/bin/travis-ci.sh -bez2 + +py3_osx_rust_tests: &py3_osx_rust_tests + <<: *base_osx_rust_tests + <<: *py3_osx_test_config + name: "OSX Rust + platform-specific tests (Py3 PEX)" + env: + - *py3_osx_test_config_env + - CACHE_NAME=macosrusttests.py3 + addons: + homebrew: + packages: *py3_osx_config_brew_packages + casks: *base_osx_rust_tests_brew_casks + script: + - ./build-support/bin/travis-ci.sh -bez + +# ------------------------------------------------------------------------- +# Test matrix +# ------------------------------------------------------------------------- + matrix: include: - - <<: *default_linux_config - <<: *native_engine_cache_config - name: "Build Linux native engine and pants.pex" - stage: *bootstrap - services: - - docker - env: - - CACHE_NAME=linuxpexbuild - - PREPARE_DEPLOY=1 # To deploy fs_util. - before_script: - - ulimit -c unlimited - script: - - docker build --rm -t travis_ci - --build-arg "TRAVIS_USER=$(id -un)" - --build-arg "TRAVIS_UID=$(id -u)" - --build-arg "TRAVIS_GROUP=$(id -gn)" - --build-arg "TRAVIS_GID=$(id -g)" - build-support/docker/travis_ci/ - # Note that: - # * We mount ${HOME} to cache the ${HOME}/.cache/pants/rust-toolchain. - # * With no args ci.sh just bootstraps a pants.pex. - # * We also build fs_util, to take advantage of the rust code built during bootstrapping. - # * We source ~/.bashrc, which is populated with pyenv initialization commands in the travis_ci image. - - docker run --rm -t - -v "${HOME}:/travis/home" - -v "${TRAVIS_BUILD_DIR}:/travis/workdir" - travis_ci:latest - sh -c "source ~/.bashrc && ./build-support/bin/ci.sh && ./build-support/bin/release.sh -f" - - aws --no-sign-request --region us-east-1 s3 cp ${TRAVIS_BUILD_DIR}/pants.pex ${BOOTSTRAPPED_PEX_URL_PREFIX}.linux - - # Build macOS engine - - <<: *default_osx_config - <<: *native_engine_cache_config - name: "Build OSX native engine and pants.pex" - stage: *bootstrap - # We request the oldest image we can (corresponding to OSX 10.11) for maximum compatibility. - # We use 10.11 as a minimum to avoid https://github.com/rust-lang/regex/issues/489. - # See: https://docs.travis-ci.com/user/reference/osx/#OS-X-Version - osx_image: xcode8 - env: - - CACHE_NAME=macospexbuild - - PREPARE_DEPLOY=1 # To deploy fs_util. - script: - # With no args ci.sh just bootstraps a pants.pex. We also build fs_util, to take advantage - # of the rust code built during bootstrapping. - - ./build-support/bin/ci.sh && ./build-support/bin/release.sh -f - - aws --no-sign-request --region us-east-1 s3 cp ${TRAVIS_BUILD_DIR}/pants.pex ${BOOTSTRAPPED_PEX_URL_PREFIX}.osx - after_failure: - - ./build-support/bin/ci-failure.sh - - - <<: *default_linux_test_config - name: "Build Linux wheels" - stage: *test - env: - - CACHE_NAME=linuxwheelsbuild - - RUN_PANTS_FROM_PEX=1 - - PREPARE_DEPLOY=1 - script: - - ./build-support/bin/release.sh -n + - <<: *py2_linux_build_engine + - <<: *py3_linux_build_engine - - <<: *default_osx_test_config - name: "Build OSX wheels" - osx_image: xcode8 - env: - - CACHE_NAME=osxwheelsbuild - - RUN_PANTS_FROM_PEX=1 - - PREPARE_DEPLOY=1 - script: - - ./build-support/bin/release.sh -n + - <<: *py2_osx_build_engine + - <<: *py3_osx_build_engine - # TODO: Update this to use 10.14 once it is available - - <<: *default_osx_test_config - name: "OSX 10.12 sanity check" - osx_image: xcode9.2 - env: - - CACHE_NAME=macos10.12sanity - script: - - MODE=debug ./build-support/bin/travis-ci.sh -bm + - <<: *py2_linux_build_wheels + - <<: *py3_linux_build_wheels - - <<: *default_osx_test_config - name: "OSX 10.13 sanity check" - osx_image: xcode10.1 - env: - - CACHE_NAME=macos10.13sanity - script: - - MODE=debug ./build-support/bin/travis-ci.sh -bm + - <<: *py2_osx_build_wheels + - <<: *py3_osx_build_wheels - # Deploy Pex Stable - - name: "Deploy stable multiplatform pants.pex" - os: linux - language: python - stage: *build_stable - env: - - CACHE_NAME=linuxpexdeploystable - - RUN_PANTS_FROM_PEX=1 - - PANTS_PEX_RELEASE=stable - script: - - ./build-support/bin/release.sh -p - deploy: - # See https://docs.travis-ci.com/user/deployment/releases/ - provider: releases - # The pantsbuild-ci-bot OAuth token, see the pantsbuild vault for details. - api_key: - secure: "u0aCsiuVGOg28YxG0sQUovuUm29kKwQfFgHbNz2TT5L+cGoHxGl4aoVOCtuwWYEtbNGmYc8/3WRS3C/jOiqQj6JEgHUzWOsnfKUObEqNhisAmXbzBbKc0wPQTL8WNK+DKFh32sD3yPYcw+a5PTLO56+o7rqlI25LK7A17WesHC4=" - file_glob: true - file: dist/deploy/pex/* - skip_cleanup: true - on: - # We only release a pex for Pants releases, which are tagged. - tags: true - repo: pantsbuild/pants - - # Deploy Pex Unstable to s3 - - name: "Deploy unstable multiplatform pants.pex" - os: linux - language: python - stage: *build_unstable - env: - - CACHE_NAME=linuxpexdeployunstable - - RUN_PANTS_FROM_PEX=1 - - PREPARE_DEPLOY=1 - script: - - ./build-support/bin/release.sh -p && mkdir -p dist/deploy/pex/ && mv dist/pants*.pex dist/deploy/pex/ + - <<: *py2_osx_10_12_sanity_check + - <<: *py3_osx_10_12_sanity_check - - <<: *default_linux_test_config - <<: *linux_with_fuse - name: "Self-checks, lint, and JVM tests" - env: - - CACHE_NAME=linuxselfchecks - script: - - ./build-support/bin/travis-ci.sh -fbmrjt + - <<: *py2_osx_10_13_sanity_check + - <<: *py3_osx_10_13_sanity_check - - <<: *default_linux_test_config - name: "Py2 - Unit tests for pants and pants-plugins" + - <<: *deploy_stable_multiplatform_pex + - <<: *deploy_unstable_multiplatform_pex + + - <<: *py2_lint + - <<: *py3_lint + + - <<: *py2_linux_test_config + name: "Unit tests for pants and pants-plugins (Py2 PEX)" + stage: *test env: + - *py2_linux_test_config_env - CACHE_NAME=linuxunittests.py2 script: - - ./build-support/bin/travis-ci.sh -blp + - ./build-support/bin/travis-ci.sh -2blp - - <<: *default_linux_test_config - name: "Py3 - Unit tests for pants and pants-plugins" + - <<: *py3_linux_test_config + name: "Unit tests for pants and pants-plugins (Py3 PEX)" env: + - *py3_linux_test_config_env - CACHE_NAME=linuxunittests.py3 script: - - ./build-support/bin/travis-ci.sh -3blp + - ./build-support/bin/travis-ci.sh -blp - - <<: *default_linux_test_config - name: "Py2 - Python contrib tests" + - <<: *py2_linux_test_config + name: "Python contrib tests (Py2 PEX)" env: + - *py2_linux_test_config_env - CACHE_NAME=linuxcontribtests.py2 script: - - ./build-support/bin/travis-ci.sh -bn + - ./build-support/bin/travis-ci.sh -2bn - - <<: *default_linux_test_config - name: "Py3 - Python contrib tests" + - <<: *py3_linux_test_config + name: "Python contrib tests (Py3 PEX)" env: + - *py3_linux_test_config_env - CACHE_NAME=linuxcontribtests.py3 script: - - ./build-support/bin/travis-ci.sh -3bn + - ./build-support/bin/travis-ci.sh -bn - - <<: *default_linux_test_config - name: "Integration tests for pants - shard 0" + - <<: *py3_linux_test_config + name: "Integration tests for pants - shard 0 (Py3 PEX)" env: + - *py3_linux_test_config_env - CACHE_NAME=integrationshard0 script: - - ./build-support/bin/travis-ci.sh -bc3 -i 0/20 + - ./build-support/bin/travis-ci.sh -bc -i 0/20 - - <<: *default_linux_test_config - name: "Integration tests for pants - shard 1" + - <<: *py3_linux_test_config + name: "Integration tests for pants - shard 1 (Py3 PEX)" env: + - *py3_linux_test_config_env - CACHE_NAME=integrationshard1 script: - - ./build-support/bin/travis-ci.sh -bc3 -i 1/20 + - ./build-support/bin/travis-ci.sh -bc -i 1/20 - - <<: *default_linux_test_config - name: "Integration tests for pants - shard 2" + - <<: *py3_linux_test_config + name: "Integration tests for pants - shard 2 (Py3 PEX)" env: + - *py3_linux_test_config_env - CACHE_NAME=integrationshard2 script: - - ./build-support/bin/travis-ci.sh -bc3 -i 2/20 + - ./build-support/bin/travis-ci.sh -bc -i 2/20 - - <<: *default_linux_test_config - name: "Integration tests for pants - shard 3" + - <<: *py3_linux_test_config + name: "Integration tests for pants - shard 3 (Py3 PEX)" env: + - *py3_linux_test_config_env - CACHE_NAME=integrationshard3 script: - - ./build-support/bin/travis-ci.sh -bc3 -i 3/20 + - ./build-support/bin/travis-ci.sh -bc -i 3/20 - - <<: *default_linux_test_config - name: "Integration tests for pants - shard 4" + - <<: *py3_linux_test_config + name: "Integration tests for pants - shard 4 (Py3 PEX)" env: + - *py3_linux_test_config_env - CACHE_NAME=integrationshard4 script: - - ./build-support/bin/travis-ci.sh -bc3 -i 4/20 + - ./build-support/bin/travis-ci.sh -bc -i 4/20 - - <<: *default_linux_test_config - name: "Integration tests for pants - shard 5" + - <<: *py3_linux_test_config + name: "Integration tests for pants - shard 5 (Py3 PEX)" env: + - *py3_linux_test_config_env - CACHE_NAME=integrationshard5 script: - - ./build-support/bin/travis-ci.sh -bc3 -i 5/20 + - ./build-support/bin/travis-ci.sh -bc -i 5/20 - - <<: *default_linux_test_config - name: "Integration tests for pants - shard 6" + - <<: *py3_linux_test_config + name: "Integration tests for pants - shard 6 (Py3 PEX)" env: + - *py3_linux_test_config_env - CACHE_NAME=integrationshard6 script: - - ./build-support/bin/travis-ci.sh -bc3 -i 6/20 + - ./build-support/bin/travis-ci.sh -bc -i 6/20 - - <<: *default_linux_test_config - name: "Integration tests for pants - shard 7" + - <<: *py3_linux_test_config + name: "Integration tests for pants - shard 7 (Py3 PEX)" env: + - *py3_linux_test_config_env - CACHE_NAME=integrationshard7 script: - - ./build-support/bin/travis-ci.sh -bc3 -i 7/20 + - ./build-support/bin/travis-ci.sh -bc -i 7/20 - - <<: *default_linux_test_config - name: "Integration tests for pants - shard 8" + - <<: *py3_linux_test_config + name: "Integration tests for pants - shard 8 (Py3 PEX)" env: + - *py3_linux_test_config_env - CACHE_NAME=integrationshard8 script: - - ./build-support/bin/travis-ci.sh -bc3 -i 8/20 + - ./build-support/bin/travis-ci.sh -bc -i 8/20 - - <<: *default_linux_test_config - name: "Integration tests for pants - shard 9" + - <<: *py3_linux_test_config + name: "Integration tests for pants - shard 9 (Py3 PEX)" env: + - *py3_linux_test_config_env - CACHE_NAME=integrationshard9 script: - - ./build-support/bin/travis-ci.sh -bc3 -i 9/20 + - ./build-support/bin/travis-ci.sh -bc -i 9/20 - - <<: *default_linux_test_config - name: "Integration tests for pants - shard 10" + - <<: *py3_linux_test_config + name: "Integration tests for pants - shard 10 (Py3 PEX)" env: + - *py3_linux_test_config_env - CACHE_NAME=integrationshard10 script: - - ./build-support/bin/travis-ci.sh -bc3 -i 10/20 + - ./build-support/bin/travis-ci.sh -bc -i 10/20 - - <<: *default_linux_test_config - name: "Integration tests for pants - shard 11" + - <<: *py3_linux_test_config + name: "Integration tests for pants - shard 11 (Py3 PEX)" env: + - *py3_linux_test_config_env - CACHE_NAME=integrationshard11 script: - - ./build-support/bin/travis-ci.sh -bc3 -i 11/20 + - ./build-support/bin/travis-ci.sh -bc -i 11/20 - - <<: *default_linux_test_config - name: "Integration tests for pants - shard 12" + - <<: *py3_linux_test_config + name: "Integration tests for pants - shard 12 (Py3 PEX)" env: + - *py3_linux_test_config_env - CACHE_NAME=integrationshard12 script: - - ./build-support/bin/travis-ci.sh -bc3 -i 12/20 + - ./build-support/bin/travis-ci.sh -bc -i 12/20 - - <<: *default_linux_test_config - name: "Integration tests for pants - shard 13" + - <<: *py3_linux_test_config + name: "Integration tests for pants - shard 13 (Py3 PEX)" env: + - *py3_linux_test_config_env - CACHE_NAME=integrationshard13 script: - - ./build-support/bin/travis-ci.sh -bc3 -i 13/20 + - ./build-support/bin/travis-ci.sh -bc -i 13/20 - - <<: *default_linux_test_config - name: "Integration tests for pants - shard 14" + - <<: *py3_linux_test_config + name: "Integration tests for pants - shard 14 (Py3 PEX)" env: + - *py3_linux_test_config_env - CACHE_NAME=integrationshard14 script: - - ./build-support/bin/travis-ci.sh -bc3 -i 14/20 + - ./build-support/bin/travis-ci.sh -bc -i 14/20 - - <<: *default_linux_test_config - name: "Integration tests for pants - shard 15" + - <<: *py3_linux_test_config + name: "Integration tests for pants - shard 15 (Py3 PEX)" env: + - *py3_linux_test_config_env - CACHE_NAME=integrationshard15 script: - - ./build-support/bin/travis-ci.sh -bc3 -i 15/20 + - ./build-support/bin/travis-ci.sh -bc -i 15/20 - - <<: *default_linux_test_config - name: "Integration tests for pants - shard 16" + - <<: *py3_linux_test_config + name: "Integration tests for pants - shard 16 (Py3 PEX)" env: + - *py3_linux_test_config_env - CACHE_NAME=integrationshard16 script: - - ./build-support/bin/travis-ci.sh -bc3 -i 16/20 + - ./build-support/bin/travis-ci.sh -bc -i 16/20 - - <<: *default_linux_test_config - name: "Integration tests for pants - shard 17" + - <<: *py3_linux_test_config + name: "Integration tests for pants - shard 17 (Py3 PEX)" env: + - *py3_linux_test_config_env - CACHE_NAME=integrationshard17 script: - - ./build-support/bin/travis-ci.sh -bc3 -i 17/20 + - ./build-support/bin/travis-ci.sh -bc -i 17/20 - - <<: *default_linux_test_config - name: "Integration tests for pants - shard 18" + - <<: *py3_linux_test_config + name: "Integration tests for pants - shard 18 (Py3 PEX)" env: + - *py3_linux_test_config_env - CACHE_NAME=integrationshard18 script: - - ./build-support/bin/travis-ci.sh -bc3 -i 18/20 + - ./build-support/bin/travis-ci.sh -bc -i 18/20 - - <<: *default_linux_test_config - name: "Integration tests for pants - shard 19" + - <<: *py3_linux_test_config + name: "Integration tests for pants - shard 19 (Py3 PEX)" env: + - *py3_linux_test_config_env - CACHE_NAME=integrationshard19 script: - - ./build-support/bin/travis-ci.sh -bc3 -i 19/20 + - ./build-support/bin/travis-ci.sh -bc -i 19/20 - - <<: *default_linux_test_config - name: "Integration tests for pants (Python 2) - shard 0" + - <<: *py2_linux_test_config + name: "Integration tests for pants - shard 0 (Py2 PEX)" stage: *cron env: + - *py2_linux_test_config_env - CACHE_NAME=cronshard0 script: - - ./build-support/bin/travis-ci.sh -bc -i 0/20 + - ./build-support/bin/travis-ci.sh -bc2 -i 0/20 - - <<: *default_linux_test_config - name: "Integration tests for pants (Python 2) - shard 1" + - <<: *py2_linux_test_config + name: "Integration tests for pants - shard 1 (Py2 PEX)" stage: *cron env: + - *py2_linux_test_config_env - CACHE_NAME=cronshard1 script: - - ./build-support/bin/travis-ci.sh -bc -i 1/20 + - ./build-support/bin/travis-ci.sh -bc2 -i 1/20 - - <<: *default_linux_test_config - name: "Integration tests for pants (Python 2) - shard 2" + - <<: *py2_linux_test_config + name: "Integration tests for pants - shard 2 (Py2 PEX)" stage: *cron env: + - *py2_linux_test_config_env - CACHE_NAME=cronshard2 script: - - ./build-support/bin/travis-ci.sh -bc -i 2/20 + - ./build-support/bin/travis-ci.sh -bc2 -i 2/20 - - <<: *default_linux_test_config - name: "Integration tests for pants (Python 2) - shard 3" + - <<: *py2_linux_test_config + name: "Integration tests for pants - shard 3 (Py2 PEX)" stage: *cron env: + - *py2_linux_test_config_env - CACHE_NAME=cronshard3 script: - - ./build-support/bin/travis-ci.sh -bc -i 3/20 + - ./build-support/bin/travis-ci.sh -bc2 -i 3/20 - - <<: *default_linux_test_config - name: "Integration tests for pants (Python 2) - shard 4" + - <<: *py2_linux_test_config + name: "Integration tests for pants - shard 4 (Py2 PEX)" stage: *cron env: + - *py2_linux_test_config_env - CACHE_NAME=cronshard4 script: - - ./build-support/bin/travis-ci.sh -bc -i 4/20 + - ./build-support/bin/travis-ci.sh -bc2 -i 4/20 - - <<: *default_linux_test_config - name: "Integration tests for pants (Python 2) - shard 5" + - <<: *py2_linux_test_config + name: "Integration tests for pants - shard 5 (Py2 PEX)" stage: *cron env: + - *py2_linux_test_config_env - CACHE_NAME=cronshard5 script: - - ./build-support/bin/travis-ci.sh -bc -i 5/20 + - ./build-support/bin/travis-ci.sh -bc2 -i 5/20 - - <<: *default_linux_test_config - name: "Integration tests for pants (Python 2) - shard 6" + - <<: *py2_linux_test_config + name: "Integration tests for pants - shard 6 (Py2 PEX)" stage: *cron env: + - *py2_linux_test_config_env - CACHE_NAME=cronshard6 script: - - ./build-support/bin/travis-ci.sh -bc -i 6/20 + - ./build-support/bin/travis-ci.sh -bc2 -i 6/20 - - <<: *default_linux_test_config - name: "Integration tests for pants (Python 2) - shard 7" + - <<: *py2_linux_test_config + name: "Integration tests for pants - shard 7 (Py2 PEX)" stage: *cron env: + - *py2_linux_test_config_env - CACHE_NAME=cronshard7 script: - - ./build-support/bin/travis-ci.sh -bc -i 7/20 + - ./build-support/bin/travis-ci.sh -bc2 -i 7/20 - - <<: *default_linux_test_config - name: "Integration tests for pants (Python 2) - shard 8" + - <<: *py2_linux_test_config + name: "Integration tests for pants - shard 8 (Py2 PEX)" stage: *cron env: + - *py2_linux_test_config_env - CACHE_NAME=cronshard8 script: - - ./build-support/bin/travis-ci.sh -bc -i 8/20 + - ./build-support/bin/travis-ci.sh -bc2 -i 8/20 - - <<: *default_linux_test_config - name: "Integration tests for pants (Python 2) - shard 9" + - <<: *py2_linux_test_config + name: "Integration tests for pants - shard 9 (Py2 PEX)" stage: *cron env: + - *py2_linux_test_config_env - CACHE_NAME=cronshard9 script: - - ./build-support/bin/travis-ci.sh -bc -i 9/20 + - ./build-support/bin/travis-ci.sh -bc2 -i 9/20 - - <<: *default_linux_test_config - name: "Integration tests for pants (Python 2) - shard 10" + - <<: *py2_linux_test_config + name: "Integration tests for pants - shard 10 (Py2 PEX)" stage: *cron env: + - *py2_linux_test_config_env - CACHE_NAME=cronshard10 script: - - ./build-support/bin/travis-ci.sh -bc -i 10/20 + - ./build-support/bin/travis-ci.sh -bc2 -i 10/20 - - <<: *default_linux_test_config - name: "Integration tests for pants (Python 2) - shard 11" + - <<: *py2_linux_test_config + name: "Integration tests for pants - shard 11 (Py2 PEX)" stage: *cron env: + - *py2_linux_test_config_env - CACHE_NAME=cronshard11 script: - - ./build-support/bin/travis-ci.sh -bc -i 11/20 + - ./build-support/bin/travis-ci.sh -bc2 -i 11/20 - - <<: *default_linux_test_config - name: "Integration tests for pants (Python 2) - shard 12" + - <<: *py2_linux_test_config + name: "Integration tests for pants - shard 12 (Py2 PEX)" stage: *cron env: + - *py2_linux_test_config_env - CACHE_NAME=cronshard12 script: - - ./build-support/bin/travis-ci.sh -bc -i 12/20 + - ./build-support/bin/travis-ci.sh -bc2 -i 12/20 - - <<: *default_linux_test_config - name: "Integration tests for pants (Python 2) - shard 13" + - <<: *py2_linux_test_config + name: "Integration tests for pants - shard 13 (Py2 PEX)" stage: *cron env: + - *py2_linux_test_config_env - CACHE_NAME=cronshard13 script: - - ./build-support/bin/travis-ci.sh -bc -i 13/20 + - ./build-support/bin/travis-ci.sh -bc2 -i 13/20 - - <<: *default_linux_test_config - name: "Integration tests for pants (Python 2) - shard 14" + - <<: *py2_linux_test_config + name: "Integration tests for pants - shard 14 (Py2 PEX)" stage: *cron env: + - *py2_linux_test_config_env - CACHE_NAME=cronshard14 script: - - ./build-support/bin/travis-ci.sh -bc -i 14/20 + - ./build-support/bin/travis-ci.sh -bc2 -i 14/20 - - <<: *default_linux_test_config - name: "Integration tests for pants (Python 2) - shard 15" + - <<: *py2_linux_test_config + name: "Integration tests for pants - shard 15 (Py2 PEX)" stage: *cron env: + - *py2_linux_test_config_env - CACHE_NAME=cronshard15 script: - - ./build-support/bin/travis-ci.sh -bc -i 15/20 + - ./build-support/bin/travis-ci.sh -bc2 -i 15/20 - - <<: *default_linux_test_config - name: "Integration tests for pants (Python 2) - shard 16" + - <<: *py2_linux_test_config + name: "Integration tests for pants - shard 16 (Py2 PEX)" stage: *cron env: + - *py2_linux_test_config_env - CACHE_NAME=cronshard16 script: - - ./build-support/bin/travis-ci.sh -bc -i 16/20 + - ./build-support/bin/travis-ci.sh -bc2 -i 16/20 - - <<: *default_linux_test_config - name: "Integration tests for pants (Python 2) - shard 17" + - <<: *py2_linux_test_config + name: "Integration tests for pants - shard 17 (Py2 PEX)" stage: *cron env: + - *py2_linux_test_config_env - CACHE_NAME=cronshard17 script: - - ./build-support/bin/travis-ci.sh -bc -i 17/20 + - ./build-support/bin/travis-ci.sh -bc2 -i 17/20 - - <<: *default_linux_test_config - name: "Integration tests for pants (Python 2) - shard 18" + - <<: *py2_linux_test_config + name: "Integration tests for pants - shard 18 (Py2 PEX)" stage: *cron env: + - *py2_linux_test_config_env - CACHE_NAME=cronshard18 script: - - ./build-support/bin/travis-ci.sh -bc -i 18/20 + - ./build-support/bin/travis-ci.sh -bc2 -i 18/20 - - <<: *default_linux_test_config - name: "Integration tests for pants (Python 2) - shard 19" + - <<: *py2_linux_test_config + name: "Integration tests for pants - shard 19 (Py2 PEX)" stage: *cron env: + - *py2_linux_test_config_env - CACHE_NAME=cronshard19 script: - - ./build-support/bin/travis-ci.sh -bc -i 19/20 + - ./build-support/bin/travis-ci.sh -bc2 -i 19/20 - # Rust on linux - - <<: *linux_with_fuse - <<: *native_engine_cache_config - name: "Linux Rust tests" - env: - - CACHE_NAME=linuxrusttests - os: linux - dist: xenial - sudo: required - stage: *test - language: python - python: *python_version - before_script: - - ulimit -c unlimited - - ulimit -n 8192 - script: - - ./build-support/bin/travis-ci.sh -be + - <<: *py2_linux_rust_tests + - <<: *py3_linux_rust_tests - # Rust on macOS - - <<: *default_osx_test_config - name: "OSX Rust + platform-specific tests" - env: - - CACHE_NAME=macosrusttests - # Fuse actually works on this image. It hangs on many others. - osx_image: xcode8.3 - addons: - homebrew: - casks: - - osxfuse - script: - - ./build-support/bin/travis-ci.sh -bez + - <<: *py2_osx_rust_tests + - <<: *py3_osx_rust_tests - # Rust Clippy on Linux - <<: *linux_with_fuse <<: *native_engine_cache_config name: "Linux Rust Clippy" env: + - *py2_linux_test_config_env - CACHE_NAME=linuxclippy os: linux dist: xenial @@ -657,19 +965,19 @@ matrix: - ulimit -c unlimited - ulimit -n 8192 script: - - ./build-support/bin/travis-ci.sh -bs + - ./build-support/bin/travis-ci.sh -bs2 - # Cargo audit - <<: *linux_with_fuse name: "Cargo audit" env: + - *py2_linux_test_config_env - CACHE_NAME=linuxcargoaudit os: linux dist: xenial sudo: required stage: *cron script: - - ./build-support/bin/travis-ci.sh -ba + - ./build-support/bin/travis-ci.sh -ba2 deploy: # Deploy whatever a previous stage has left in dist/deploy. @@ -689,4 +997,3 @@ deploy: # release branches; eg `1.3.x` all_branches: true repo: pantsbuild/pants - diff --git a/build-support/bin/ci.sh b/build-support/bin/ci.sh index 46a922941872..88ea039cd03b 100755 --- a/build-support/bin/ci.sh +++ b/build-support/bin/ci.sh @@ -2,7 +2,7 @@ # We use some subshell pipelines to collect target lists, make sure target collection failing # fails the build. -set -o pipefail +set -ox pipefail REPO_ROOT=$(cd $(dirname "${BASH_SOURCE[0]}") && cd "$(git rev-parse --show-toplevel)" && pwd) cd ${REPO_ROOT} @@ -13,10 +13,9 @@ function usage() { cat <> /travis/home/.bashrc 'eval "$(pyenv init -)"' RUN echo >> /travis/home/.bashrc 'eval "$(pyenv virtualenv-init -)"' +# ----------------------------------------------------------------------- +# This section belongs in centos6, but is here temporarily because +# centos6's build must first be published to Docker. + +RUN mkdir -p /python-3-docker-build/python-3.6.8-{source,install} +RUN curl -L -v https://www.python.org/ftp/python/3.6.8/Python-3.6.8.tgz \ + | tar -zxvC /python-3-docker-build/python-3.6.8-source +RUN ls -lAvF /python-3-docker-build/python-3.6.8-source +RUN /usr/bin/scl enable devtoolset-7 -- \ + bash -c 'pushd /python-3-docker-build/python-3.6.8-source/Python-3.6.8 \ + && ./configure --prefix=/python-3-docker-build/python-3.6.8-install \ + --enable-optimizations \ + && make install' +ENV PATH "${PATH}:/python-3-docker-build/python-3.6.8-install/bin" +RUN ls -lAvF /python-3-docker-build/python-3.6.8-install/bin + +# ----------------------------------------------------------------------- + # Setup mount points for the travis ci user & workdir. VOLUME /travis/home VOLUME /travis/workdir @@ -42,9 +60,6 @@ RUN groupadd --gid ${TRAVIS_GID} ${TRAVIS_GROUP} || true RUN useradd -d /travis/home -g ${TRAVIS_GROUP} --uid ${TRAVIS_UID} ${TRAVIS_USER} USER ${TRAVIS_USER}:${TRAVIS_GROUP} -# Ensure Pants runs under the 2.7.13 interpreter. -ENV PANTS_PYTHON_SETUP_INTERPRETER_CONSTRAINTS="['CPython==2.7.13']" - # Our newly created user is unlikely to have a sane environment: set a locale at least. ENV LC_ALL="en_US.UTF-8" diff --git a/build-support/travis/generate_travis_yml.py b/build-support/travis/generate_travis_yml.py index ee9e822e5bc5..2cc19cd99149 100644 --- a/build-support/travis/generate_travis_yml.py +++ b/build-support/travis/generate_travis_yml.py @@ -15,6 +15,9 @@ # GENERATED, DO NOT EDIT! # To change, edit build-support/travis/travis.yml.mustache and run # ./pants --quiet run build-support/travis:generate_travis_yml > .travis.yml +# +# Tip: Copy the generated `.travis.yml` into https://yamlvalidator.com to validate the YAML +# and see how the entries resolve to normalized JSON (helpful to debug anchors). """ diff --git a/build-support/travis/travis.yml.mustache b/build-support/travis/travis.yml.mustache index d046a7848236..8845bee41f0d 100644 --- a/build-support/travis/travis.yml.mustache +++ b/build-support/travis/travis.yml.mustache @@ -3,6 +3,10 @@ # Conditions are documented here: https://docs.travis-ci.com/user/conditions-v1 conditions: v1 +# ------------------------------------------------------------------------- +# Global setup +# ------------------------------------------------------------------------- + env: global: - PANTS_CONFIG_FILES="${TRAVIS_BUILD_DIR}/pants.travis-ci.ini" @@ -24,6 +28,10 @@ stages: - name: &build_unstable Deploy Pants Pex Unstable if: tag IS NOT present AND type NOT IN (pull_request, cron) +# ------------------------------------------------------------------------- +# Cache config +# ------------------------------------------------------------------------- + # Travis cache config for jobs that build the native engine. native_engine_cache_config: &native_engine_cache_config before_cache: @@ -75,7 +83,11 @@ pants_run_cache_config: &pants_run_cache_config # https://github.com/pantsbuild/pants/issues/2485 - ${HOME}/.npm -default_linux_config: &default_linux_config +# ------------------------------------------------------------------------- +# Generic shard setups +# ------------------------------------------------------------------------- + +base_linux_config: &base_linux_config os: linux dist: xenial sudo: required @@ -92,35 +104,87 @@ default_linux_config: &default_linux_config - jq - unzip language: python - python: &python_version "2.7" before_install: - ./build-support/bin/install_aws_cli_for_ci.sh after_failure: - ./build-support/bin/ci-failure.sh -default_linux_test_config: &default_linux_test_config - <<: *default_linux_config +py2_linux_config: &py2_linux_config + <<: *base_linux_config + python: &python_version "2.7" + +py3_linux_config: &py3_linux_config + <<: *base_linux_config + python: &python_version "3.6" + +base_linux_test_config: &base_linux_test_config + <<: *base_linux_config <<: *pants_run_cache_config - stage: *test before_install: {{>before_install_linux}} before_script: - - ./build-support/bin/get_ci_bootstrapped_pants_pex.sh ${BOOTSTRAPPED_PEX_BUCKET} ${BOOTSTRAPPED_PEX_KEY_PREFIX}.linux + - ./build-support/bin/get_ci_bootstrapped_pants_pex.sh ${BOOTSTRAPPED_PEX_BUCKET} ${BOOTSTRAPPED_PEX_KEY_PREFIX}.${BOOTSTRAPPED_PEX_KEY_SUFFIX} -default_osx_config: &default_osx_config +py2_linux_test_config: &py2_linux_test_config + <<: *py2_linux_config + <<: *base_linux_test_config + stage: *cron + env: + - &py2_linux_test_config_env BOOTSTRAPPED_PEX_KEY_SUFFIX=py2.linux + +py3_linux_test_config: &py3_linux_test_config + <<: *py3_linux_config + <<: *base_linux_test_config + stage: *test + env: + - &py3_linux_test_config_env BOOTSTRAPPED_PEX_KEY_SUFFIX=py3.linux + +base_osx_config: &base_osx_config os: osx language: generic before_install: {{>before_install_osx}} -default_osx_test_config: &default_osx_test_config - <<: *default_osx_config +py2_osx_config: &py2_osx_config + <<: *base_osx_config + +py3_osx_config: &py3_osx_config + <<: *base_osx_config + addons: + brew: + packages: &py3_osx_config_brew_packages + - pyenv + - openssl + env: + # Fix Python 3 issue linking to OpenSSL + - PATH="/usr/local/opt/openssl/bin:$PATH" LDFLAGS="-L/usr/local/opt/openssl/lib" CPPFLAGS="-I/usr/local/opt/openssl/include" + before_install: + {{>before_install_osx}} + # Install pyenv the conventional way, instead of through addon, due to execution order + - brew install pyenv + - pyenv install 3.6.8 && pyenv global 3.6.8 + +base_osx_test_config: &base_osx_test_config <<: *pants_run_cache_config - stage: *test before_script: - ulimit -c unlimited - ulimit -n 8192 - - ./build-support/bin/get_ci_bootstrapped_pants_pex.sh ${BOOTSTRAPPED_PEX_BUCKET} ${BOOTSTRAPPED_PEX_KEY_PREFIX}.osx + - ./build-support/bin/get_ci_bootstrapped_pants_pex.sh ${BOOTSTRAPPED_PEX_BUCKET} ${BOOTSTRAPPED_PEX_KEY_PREFIX}.${BOOTSTRAPPED_PEX_KEY_SUFFIX} + +py2_osx_test_config: &py2_osx_test_config + <<: *py2_osx_config + <<: *base_osx_test_config + stage: *cron + env: + - &py2_osx_test_config_env BOOTSTRAPPED_PEX_KEY_SUFFIX=py2.osx + +py3_osx_test_config: &py3_osx_test_config + <<: *py3_osx_config + <<: *base_osx_test_config + stage: *test + env: + # Must duplicate OpenSSL config from py3_osx_config's env because it cannot be merged into a new anchor + - &py3_osx_test_config_env PATH="/usr/local/opt/openssl/bin:$PATH" LDFLAGS="-L/usr/local/opt/openssl/lib" CPPFLAGS="-I/usr/local/opt/openssl/include" BOOTSTRAPPED_PEX_KEY_SUFFIX=py3.osx linux_with_fuse: &linux_with_fuse before_install: @@ -130,225 +194,427 @@ linux_with_fuse: &linux_with_fuse - sudo chmod 666 /dev/fuse - sudo chown root:$USER /etc/fuse.conf +# ------------------------------------------------------------------------- +# Bootstrap engine shards +# ------------------------------------------------------------------------- + +base_linux_build_engine: &base_linux_build_engine + <<: *native_engine_cache_config + stage: *bootstrap + services: + - docker + env: + - &base_linux_build_engine_env PREPARE_DEPLOY=1 # To deploy fs_util. + before_script: + - ulimit -c unlimited + script: + - docker build --rm -t travis_ci + --build-arg "TRAVIS_USER=$(id -un)" + --build-arg "TRAVIS_UID=$(id -u)" + --build-arg "TRAVIS_GROUP=$(id -gn)" + --build-arg "TRAVIS_GID=$(id -g)" + build-support/docker/travis_ci/ + # Note that: + # * We mount ${HOME} to cache the ${HOME}/.cache/pants/rust-toolchain. + # * With no args ci.sh just bootstraps a pants.pex. + # * We also build fs_util, to take advantage of the rust code built during bootstrapping. + - docker run --rm -t + -v "${HOME}:/travis/home" + -v "${TRAVIS_BUILD_DIR}:/travis/workdir" + travis_ci:latest + sh -c "source ~/.bashrc && ./build-support/bin/ci.sh ${BOOTSTRAP_ARGS} && ./build-support/bin/release.sh -f" + - aws --no-sign-request --region us-east-1 s3 cp ${TRAVIS_BUILD_DIR}/pants.pex ${BOOTSTRAPPED_PEX_URL_PREFIX}.${BOOTSTRAPPED_PEX_KEY_SUFFIX} + +py2_linux_build_engine: &py2_linux_build_engine + <<: *py2_linux_config + <<: *base_linux_build_engine + name: "Build Linux native engine and pants.pex (Py2 PEX)" + env: + - *base_linux_build_engine_env + - CACHE_NAME=linuxpexbuild.py2 + - BOOTSTRAPPED_PEX_KEY_SUFFIX=py2.linux + - BOOTSTRAP_ARGS='-2' + +py3_linux_build_engine: &py3_linux_build_engine + <<: *py3_linux_config + <<: *base_linux_build_engine + name: "Build Linux native engine and pants.pex (Py3 PEX)" + env: + - *base_linux_build_engine_env + - CACHE_NAME=linuxpexbuild.py3 + - BOOTSTRAPPED_PEX_KEY_SUFFIX=py3.linux + - BOOTSTRAP_ARGS='' + +base_osx_build_engine: &base_osx_build_engine + <<: *native_engine_cache_config + stage: *bootstrap + # We request the oldest image we can (corresponding to OSX 10.11) for maximum compatibility. + # We use 10.11 as a minimum to avoid https://github.com/rust-lang/regex/issues/489. + # See: https://docs.travis-ci.com/user/reference/osx/#OS-X-Version + osx_image: xcode8 + env: + - &base_osx_build_engine_env PREPARE_DEPLOY=1 # To deploy fs_util. + script: + # With no args ci.sh just bootstraps a pants.pex. We also build fs_util, to take advantage + # of the rust code built during bootstrapping. + - ./build-support/bin/ci.sh ${BOOTSTRAP_ARGS} && ./build-support/bin/release.sh -f + - aws --no-sign-request --region us-east-1 s3 cp ${TRAVIS_BUILD_DIR}/pants.pex ${BOOTSTRAPPED_PEX_URL_PREFIX}.${BOOTSTRAPPED_PEX_KEY_SUFFIX} + after_failure: + - ./build-support/bin/ci-failure.sh + +py2_osx_build_engine: &py2_osx_build_engine + <<: *py2_osx_config + <<: *base_osx_build_engine + name: "Build OSX native engine and pants.pex (Py2 PEX)" + env: + - *base_osx_build_engine_env + - CACHE_NAME=osxpexbuild.py2 + - BOOTSTRAPPED_PEX_KEY_SUFFIX=py2.osx + - BOOTSTRAP_ARGS='-2' + +py3_osx_build_engine: &py3_osx_build_engine + <<: *py3_osx_config + <<: *base_osx_build_engine + name: "Build OSX native engine and pants.pex (Py3 PEX)" + + env: + - *base_osx_build_engine_env + - CACHE_NAME=osxpexbuild.py3 + - BOOTSTRAPPED_PEX_KEY_SUFFIX=py3.osx + - BOOTSTRAP_ARGS='' + +# ------------------------------------------------------------------------- +# Build wheels +# ------------------------------------------------------------------------- + +base_build_wheels: &base_build_wheels + env: + - &base_build_wheels_env RUN_PANTS_FROM_PEX=1 PREPARE_DEPLOY=1 + script: + - ./build-support/bin/release.sh -n + +py2_linux_build_wheels: &py2_linux_build_wheels + <<: *py2_linux_test_config + <<: *base_build_wheels + stage: *cron + name: "Build Linux wheels (Py2 PEX)" + env: + - *py2_linux_test_config_env + - *base_build_wheels_env + - CACHE_NAME=linuxwheelsbuild.py2 + +py3_linux_build_wheels: &py3_linux_build_wheels + <<: *py3_linux_test_config + <<: *base_build_wheels + name: "Build Linux wheels (Py3 PEX)" + env: + - *py3_linux_test_config_env + - *base_build_wheels_env + - CACHE_NAME=linuxwheelsbuild.py3 + +py2_osx_build_wheels: &py2_osx_build_wheels + <<: *py2_osx_test_config + <<: *base_build_wheels + name: "Build OSX wheels (Py2 PEX)" + stage: *cron + env: + - *py2_osx_test_config_env + - *base_build_wheels_env + - CACHE_NAME=osxwheelsbuild.py2 + +py3_osx_build_wheels: &py3_osx_build_wheels + <<: *py3_osx_test_config + <<: *base_build_wheels + name: "Build OSX wheels (Py3 PEX)" + env: + - *py3_osx_test_config_env + - *base_build_wheels_env + - CACHE_NAME=osxwheelsbuild.py3 + +# ------------------------------------------------------------------------- +# OSX sanity checks +# ------------------------------------------------------------------------- + +base_osx_sanity_check: &base_osx_sanity_check + script: + - MODE=debug ./build-support/bin/travis-ci.sh -bm + +# TODO: Update this to use 10.14 once it is available +base_osx_10_12_sanity_check: &base_osx_10_12_sanity_check + <<: *base_osx_sanity_check + osx_image: xcode9.2 + +py2_osx_10_12_sanity_check: &py2_osx_10_12_sanity_check + <<: *py2_osx_test_config + <<: *base_osx_10_12_sanity_check + name: "OSX 10.12 sanity check (Py2 PEX)" + stage: *cron + env: + - *py2_osx_test_config_env + - CACHE_NAME=macos10.12sanity.py2 + +py3_osx_10_12_sanity_check: &py3_osx_10_12_sanity_check + <<: *py3_osx_test_config + <<: *base_osx_10_12_sanity_check + name: "OSX 10.12 sanity check (Py3 PEX)" + env: + - *py3_osx_test_config_env + - CACHE_NAME=macos10.12sanity.py3 + +base_osx_10_13_sanity_check: &base_osx_10_13_sanity_check + <<: *base_osx_sanity_check + osx_image: xcode10.1 + +py2_osx_10_13_sanity_check: &py2_osx_10_13_sanity_check + <<: *py2_osx_test_config + <<: *base_osx_10_13_sanity_check + name: "OSX 10.13 sanity check (Py2 PEX)" + stage: *cron + env: + - *py2_osx_test_config_env + - CACHE_NAME=macos10.13sanity.py2 + +py3_osx_10_13_sanity_check: &py3_osx_10_13_sanity_check + <<: *py3_osx_test_config + <<: *base_osx_10_13_sanity_check + name: "OSX 10.13 sanity check (Py3 PEX)" + env: + - *py3_osx_test_config_env + - CACHE_NAME=macos10.13sanity.py3 + +# ------------------------------------------------------------------------- +# Lint +# ------------------------------------------------------------------------- + +base_lint: &base_lint + <<: *linux_with_fuse + script: + - ./build-support/bin/travis-ci.sh -fbmrjt + +py2_lint: &py2_lint + <<: *py2_linux_test_config + <<: *base_lint + name: "Self-checks, lint, and JVM tests (Py2 PEX)" + env: + - *py2_linux_test_config_env + - CACHE_NAME=linuxselfchecks.py2 + +py3_lint: &py3_lint + <<: *py3_linux_test_config + <<: *base_lint + name: "Self-checks, lint, and JVM tests (Py3 PEX)" + env: + - *py3_linux_test_config_env + - CACHE_NAME=linuxselfchecks.py3 + +# ------------------------------------------------------------------------- +# Deploy +# ------------------------------------------------------------------------- + +deploy_stable_multiplatform_pex: &deploy_stable_multiplatform_pex + name: "Deploy stable multiplatform pants.pex" + os: linux + language: python + stage: *build_stable + env: + - CACHE_NAME=linuxpexdeploystable + - RUN_PANTS_FROM_PEX=1 + - PANTS_PEX_RELEASE=stable + script: + - ./build-support/bin/release.sh -p + deploy: + # See https://docs.travis-ci.com/user/deployment/releases/ + provider: releases + # The pantsbuild-ci-bot OAuth token, see the pantsbuild vault for details. + api_key: + secure: "u0aCsiuVGOg28YxG0sQUovuUm29kKwQfFgHbNz2TT5L+cGoHxGl4aoVOCtuwWYEtbNGmYc8/3WRS3C/jOiqQj6JEgHUzWOsnfKUObEqNhisAmXbzBbKc0wPQTL8WNK+DKFh32sD3yPYcw+a5PTLO56+o7rqlI25LK7A17WesHC4=" + file_glob: true + file: dist/deploy/pex/* + skip_cleanup: true + on: + # We only release a pex for Pants releases, which are tagged. + tags: true + repo: pantsbuild/pants + +deploy_unstable_multiplatform_pex: &deploy_unstable_multiplatform_pex + name: "Deploy unstable multiplatform pants.pex" + os: linux + language: python + stage: *build_unstable + env: + - CACHE_NAME=linuxpexdeployunstable + - RUN_PANTS_FROM_PEX=1 + - PREPARE_DEPLOY=1 + script: + - ./build-support/bin/release.sh -p && mkdir -p dist/deploy/pex/ && mv dist/pants*.pex dist/deploy/pex/ + +# ------------------------------------------------------------------------- +# Rust +# ------------------------------------------------------------------------- + +base_linux_rust_tests: &base_linux_rust_tests + <<: *linux_with_fuse + <<: *native_engine_cache_config + env: + - CACHE_NAME=linuxrusttests + os: linux + dist: xenial + sudo: required + language: python + before_script: + - ulimit -c unlimited + - ulimit -n 8192 + +py2_linux_rust_tests: &py2_linux_rust_tests + <<: *base_linux_rust_tests + name: "Linux Rust tests (Py2 PEX)" + python: 2.7 + stage: *cron + env: + - *py2_linux_test_config_env + - CACHE_NAME=linuxrusttests.py2 + script: + - ./build-support/bin/travis-ci.sh -be2 + +py3_linux_rust_tests: &py3_linux_rust_tests + <<: *base_linux_rust_tests + name: "Linux Rust tests (Py3 PEX)" + python: 3.6 + stage: *test + env: + - *py3_linux_test_config_env + - CACHE_NAME=linuxrusttests.py3 + script: + - ./build-support/bin/travis-ci.sh -be + +base_osx_rust_tests: &base_osx_rust_tests + # Fuse actually works on this image. It hangs on many others. + osx_image: xcode8.3 + addons: + homebrew: + casks: &base_osx_rust_tests_brew_casks + - osxfuse + +py2_osx_rust_tests: &py2_osx_rust_tests + <<: *base_osx_rust_tests + <<: *py2_osx_test_config + name: "OSX Rust + platform-specific tests (Py2 PEX)" + env: + - *py2_osx_test_config_env + - CACHE_NAME=macosrusttests.py2 + script: + - ./build-support/bin/travis-ci.sh -bez2 + +py3_osx_rust_tests: &py3_osx_rust_tests + <<: *base_osx_rust_tests + <<: *py3_osx_test_config + name: "OSX Rust + platform-specific tests (Py3 PEX)" + env: + - *py3_osx_test_config_env + - CACHE_NAME=macosrusttests.py3 + addons: + homebrew: + packages: *py3_osx_config_brew_packages + casks: *base_osx_rust_tests_brew_casks + script: + - ./build-support/bin/travis-ci.sh -bez + +# ------------------------------------------------------------------------- +# Test matrix +# ------------------------------------------------------------------------- + matrix: include: - - <<: *default_linux_config - <<: *native_engine_cache_config - name: "Build Linux native engine and pants.pex" - stage: *bootstrap - services: - - docker - env: - - CACHE_NAME=linuxpexbuild - - PREPARE_DEPLOY=1 # To deploy fs_util. - before_script: - - ulimit -c unlimited - script: - - docker build --rm -t travis_ci - --build-arg "TRAVIS_USER=$(id -un)" - --build-arg "TRAVIS_UID=$(id -u)" - --build-arg "TRAVIS_GROUP=$(id -gn)" - --build-arg "TRAVIS_GID=$(id -g)" - build-support/docker/travis_ci/ - # Note that: - # * We mount ${HOME} to cache the ${HOME}/.cache/pants/rust-toolchain. - # * With no args ci.sh just bootstraps a pants.pex. - # * We also build fs_util, to take advantage of the rust code built during bootstrapping. - # * We source ~/.bashrc, which is populated with pyenv initialization commands in the travis_ci image. - - docker run --rm -t - -v "${HOME}:/travis/home" - -v "${TRAVIS_BUILD_DIR}:/travis/workdir" - travis_ci:latest - sh -c "source ~/.bashrc && ./build-support/bin/ci.sh && ./build-support/bin/release.sh -f" - - aws --no-sign-request --region us-east-1 s3 cp ${TRAVIS_BUILD_DIR}/pants.pex ${BOOTSTRAPPED_PEX_URL_PREFIX}.linux - - # Build macOS engine - - <<: *default_osx_config - <<: *native_engine_cache_config - name: "Build OSX native engine and pants.pex" - stage: *bootstrap - # We request the oldest image we can (corresponding to OSX 10.11) for maximum compatibility. - # We use 10.11 as a minimum to avoid https://github.com/rust-lang/regex/issues/489. - # See: https://docs.travis-ci.com/user/reference/osx/#OS-X-Version - osx_image: xcode8 - env: - - CACHE_NAME=macospexbuild - - PREPARE_DEPLOY=1 # To deploy fs_util. - script: - # With no args ci.sh just bootstraps a pants.pex. We also build fs_util, to take advantage - # of the rust code built during bootstrapping. - - ./build-support/bin/ci.sh && ./build-support/bin/release.sh -f - - aws --no-sign-request --region us-east-1 s3 cp ${TRAVIS_BUILD_DIR}/pants.pex ${BOOTSTRAPPED_PEX_URL_PREFIX}.osx - after_failure: - - ./build-support/bin/ci-failure.sh - - - <<: *default_linux_test_config - name: "Build Linux wheels" - stage: *test - env: - - CACHE_NAME=linuxwheelsbuild - - RUN_PANTS_FROM_PEX=1 - - PREPARE_DEPLOY=1 - script: - - ./build-support/bin/release.sh -n + - <<: *py2_linux_build_engine + - <<: *py3_linux_build_engine - - <<: *default_osx_test_config - name: "Build OSX wheels" - osx_image: xcode8 - env: - - CACHE_NAME=osxwheelsbuild - - RUN_PANTS_FROM_PEX=1 - - PREPARE_DEPLOY=1 - script: - - ./build-support/bin/release.sh -n + - <<: *py2_osx_build_engine + - <<: *py3_osx_build_engine - # TODO: Update this to use 10.14 once it is available - - <<: *default_osx_test_config - name: "OSX 10.12 sanity check" - osx_image: xcode9.2 - env: - - CACHE_NAME=macos10.12sanity - script: - - MODE=debug ./build-support/bin/travis-ci.sh -bm + - <<: *py2_linux_build_wheels + - <<: *py3_linux_build_wheels - - <<: *default_osx_test_config - name: "OSX 10.13 sanity check" - osx_image: xcode10.1 - env: - - CACHE_NAME=macos10.13sanity - script: - - MODE=debug ./build-support/bin/travis-ci.sh -bm + - <<: *py2_osx_build_wheels + - <<: *py3_osx_build_wheels - # Deploy Pex Stable - - name: "Deploy stable multiplatform pants.pex" - os: linux - language: python - stage: *build_stable - env: - - CACHE_NAME=linuxpexdeploystable - - RUN_PANTS_FROM_PEX=1 - - PANTS_PEX_RELEASE=stable - script: - - ./build-support/bin/release.sh -p - deploy: - # See https://docs.travis-ci.com/user/deployment/releases/ - provider: releases - # The pantsbuild-ci-bot OAuth token, see the pantsbuild vault for details. - api_key: - secure: "u0aCsiuVGOg28YxG0sQUovuUm29kKwQfFgHbNz2TT5L+cGoHxGl4aoVOCtuwWYEtbNGmYc8/3WRS3C/jOiqQj6JEgHUzWOsnfKUObEqNhisAmXbzBbKc0wPQTL8WNK+DKFh32sD3yPYcw+a5PTLO56+o7rqlI25LK7A17WesHC4=" - file_glob: true - file: dist/deploy/pex/* - skip_cleanup: true - on: - # We only release a pex for Pants releases, which are tagged. - tags: true - repo: pantsbuild/pants - - # Deploy Pex Unstable to s3 - - name: "Deploy unstable multiplatform pants.pex" - os: linux - language: python - stage: *build_unstable - env: - - CACHE_NAME=linuxpexdeployunstable - - RUN_PANTS_FROM_PEX=1 - - PREPARE_DEPLOY=1 - script: - - ./build-support/bin/release.sh -p && mkdir -p dist/deploy/pex/ && mv dist/pants*.pex dist/deploy/pex/ + - <<: *py2_osx_10_12_sanity_check + - <<: *py3_osx_10_12_sanity_check - - <<: *default_linux_test_config - <<: *linux_with_fuse - name: "Self-checks, lint, and JVM tests" - env: - - CACHE_NAME=linuxselfchecks - script: - - ./build-support/bin/travis-ci.sh -fbmrjt + - <<: *py2_osx_10_13_sanity_check + - <<: *py3_osx_10_13_sanity_check + + - <<: *deploy_stable_multiplatform_pex + - <<: *deploy_unstable_multiplatform_pex - - <<: *default_linux_test_config - name: "Py2 - Unit tests for pants and pants-plugins" + - <<: *py2_lint + - <<: *py3_lint + + - <<: *py2_linux_test_config + name: "Unit tests for pants and pants-plugins (Py2 PEX)" + stage: *test env: + - *py2_linux_test_config_env - CACHE_NAME=linuxunittests.py2 script: - - ./build-support/bin/travis-ci.sh -blp + - ./build-support/bin/travis-ci.sh -2blp - - <<: *default_linux_test_config - name: "Py3 - Unit tests for pants and pants-plugins" + - <<: *py3_linux_test_config + name: "Unit tests for pants and pants-plugins (Py3 PEX)" env: + - *py3_linux_test_config_env - CACHE_NAME=linuxunittests.py3 script: - - ./build-support/bin/travis-ci.sh -3blp + - ./build-support/bin/travis-ci.sh -blp - - <<: *default_linux_test_config - name: "Py2 - Python contrib tests" + - <<: *py2_linux_test_config + name: "Python contrib tests (Py2 PEX)" env: + - *py2_linux_test_config_env - CACHE_NAME=linuxcontribtests.py2 script: - - ./build-support/bin/travis-ci.sh -bn + - ./build-support/bin/travis-ci.sh -2bn - - <<: *default_linux_test_config - name: "Py3 - Python contrib tests" + - <<: *py3_linux_test_config + name: "Python contrib tests (Py3 PEX)" env: + - *py3_linux_test_config_env - CACHE_NAME=linuxcontribtests.py3 script: - - ./build-support/bin/travis-ci.sh -3bn + - ./build-support/bin/travis-ci.sh -bn {{#integration_shards}} - - <<: *default_linux_test_config - name: "Integration tests for pants - shard {{.}}" + - <<: *py3_linux_test_config + name: "Integration tests for pants - shard {{.}} (Py3 PEX)" env: + - *py3_linux_test_config_env - CACHE_NAME=integrationshard{{.}} script: - - ./build-support/bin/travis-ci.sh -bc3 -i {{.}}/{{integration_shards_length}} + - ./build-support/bin/travis-ci.sh -bc -i {{.}}/{{integration_shards_length}} {{/integration_shards}} {{#integration_shards}} - - <<: *default_linux_test_config - name: "Integration tests for pants (Python 2) - shard {{.}}" + - <<: *py2_linux_test_config + name: "Integration tests for pants - shard {{.}} (Py2 PEX)" stage: *cron env: + - *py2_linux_test_config_env - CACHE_NAME=cronshard{{.}} script: - - ./build-support/bin/travis-ci.sh -bc -i {{.}}/{{integration_shards_length}} + - ./build-support/bin/travis-ci.sh -bc2 -i {{.}}/{{integration_shards_length}} {{/integration_shards}} - # Rust on linux - - <<: *linux_with_fuse - <<: *native_engine_cache_config - name: "Linux Rust tests" - env: - - CACHE_NAME=linuxrusttests - os: linux - dist: xenial - sudo: required - stage: *test - language: python - python: *python_version - before_script: - - ulimit -c unlimited - - ulimit -n 8192 - script: - - ./build-support/bin/travis-ci.sh -be + - <<: *py2_linux_rust_tests + - <<: *py3_linux_rust_tests - # Rust on macOS - - <<: *default_osx_test_config - name: "OSX Rust + platform-specific tests" - env: - - CACHE_NAME=macosrusttests - # Fuse actually works on this image. It hangs on many others. - osx_image: xcode8.3 - addons: - homebrew: - casks: - - osxfuse - script: - - ./build-support/bin/travis-ci.sh -bez + - <<: *py2_osx_rust_tests + - <<: *py3_osx_rust_tests - # Rust Clippy on Linux - <<: *linux_with_fuse <<: *native_engine_cache_config name: "Linux Rust Clippy" env: + - *py2_linux_test_config_env - CACHE_NAME=linuxclippy os: linux dist: xenial @@ -360,19 +626,19 @@ matrix: - ulimit -c unlimited - ulimit -n 8192 script: - - ./build-support/bin/travis-ci.sh -bs + - ./build-support/bin/travis-ci.sh -bs2 - # Cargo audit - <<: *linux_with_fuse name: "Cargo audit" env: + - *py2_linux_test_config_env - CACHE_NAME=linuxcargoaudit os: linux dist: xenial sudo: required stage: *cron script: - - ./build-support/bin/travis-ci.sh -ba + - ./build-support/bin/travis-ci.sh -ba2 deploy: # Deploy whatever a previous stage has left in dist/deploy. diff --git a/contrib/confluence/src/python/pants/contrib/confluence/util/confluence_util.py b/contrib/confluence/src/python/pants/contrib/confluence/util/confluence_util.py index fa35978048d4..1d40c573fef9 100644 --- a/contrib/confluence/src/python/pants/contrib/confluence/util/confluence_util.py +++ b/contrib/confluence/src/python/pants/contrib/confluence/util/confluence_util.py @@ -171,7 +171,7 @@ def addattachment(self, page, filename): except (IOError, OSError) as e: log.error('Failed to read data from file %s: %s' % (filename, str(e))) return None - except XMLRPCError as e: + except XMLRPCError: log.error('Failed to add file attachment %s to page: %s' % (filename, page.get('title', '[unknown title]'))) return None diff --git a/contrib/go/src/python/pants/contrib/go/tasks/go_binary_fingerprint_strategy.py b/contrib/go/src/python/pants/contrib/go/tasks/go_binary_fingerprint_strategy.py index 4cb3a0bbb170..56cc75129a3d 100644 --- a/contrib/go/src/python/pants/contrib/go/tasks/go_binary_fingerprint_strategy.py +++ b/contrib/go/src/python/pants/contrib/go/tasks/go_binary_fingerprint_strategy.py @@ -32,7 +32,7 @@ def compute_fingerprint(self, target): return fp hasher = hashlib.sha1() - hasher.update(fp) + hasher.update(fp.encode('utf-8')) hasher.update(str(self._get_build_flags_func(target)).encode('utf-8')) return hasher.hexdigest() if PY3 else hasher.hexdigest().decode('utf-8') diff --git a/contrib/scrooge/src/python/pants/contrib/scrooge/tasks/java_thrift_library_fingerprint_strategy.py b/contrib/scrooge/src/python/pants/contrib/scrooge/tasks/java_thrift_library_fingerprint_strategy.py index de5c5ff3f615..d78b20435e0e 100644 --- a/contrib/scrooge/src/python/pants/contrib/scrooge/tasks/java_thrift_library_fingerprint_strategy.py +++ b/contrib/scrooge/src/python/pants/contrib/scrooge/tasks/java_thrift_library_fingerprint_strategy.py @@ -39,7 +39,7 @@ def compute_fingerprint(self, target): default_java_namespace = self._thrift_defaults.default_java_namespace(target) if default_java_namespace: - hasher.update(default_java_namespace) + hasher.update(default_java_namespace.encode('utf-8')) if target.include_paths: hasher.update(str(target.include_paths).encode('utf-8'))