Skip to content
Permalink
Browse files

Setup UCS2 vs UCS4 travis shards

We must now build pantsbuild.pants with both unicode versions for Py2. So, we introduce Py2 to do this.

We use Pyenv to install interpreter with the relevant encoding where necessary. See https://stackoverflow.com/questions/38928942/build-python-as-ucs-4-via-pyenv.
  • Loading branch information...
Eric-Arellano committed Feb 12, 2019
1 parent a02dde1 commit 1e83f37ccd4ea9402b81df27d5dc712684aa18d5
Showing with 159 additions and 20 deletions.
  1. +63 −10 .travis.yml
  2. +35 −0 build-support/docker/travis_ci_py27_ucs2/Dockerfile
  3. +61 −10 build-support/travis/travis.yml.mustache
@@ -381,22 +381,51 @@ cargo_audit: &cargo_audit
# Build wheels
# -------------------------------------------------------------------------

# N.B. With Python 2, we must build pantsbuild.pants with both UCS2 and UCS4 to provide full
# compatibility for end users. This is because we constrain our ABI due to the native engine.
# See https://www.python.org/dev/peps/pep-0513/#ucs-2-vs-ucs-4-builds. Note this distinction is
# not necessary with Python 3.3+ due to flexible storage of Unicode strings (https://www.python.org/dev/peps/pep-0393/).

base_build_wheels: &base_build_wheels
stage: *test
env:
- &base_build_wheels_env RUN_PANTS_FROM_PEX=1 PREPARE_DEPLOY=1

linux_build_wheels: &linux_build_wheels
py27_linux_build_wheels_no_ucs: &py27_linux_build_wheels_no_ucs
# Similar to the bootstrap shard, we build Linux wheels in a docker image to maximize
# compatibility. This is a Py2.7 shard, so it is not subject to #6985.
<<: *travis_docker_image
<<: *py27_linux_test_config
<<: *base_build_wheels
name: "Build Linux wheels (No PEX)"

py27_linux_build_wheels_ucs2: &py27_linux_build_wheels_ucs2
<<: *py27_linux_build_wheels_no_ucs
name: "Build Linux wheels (Py27 and UCS2 pantsbuild.pants)"
env:
- *py27_linux_test_config_env
- *base_build_wheels_env
- CACHE_NAME=linuxwheelsbuild.ucs2
script:
- docker build --rm -t travis_ci_py27_ucs2
--build-arg "TRAVIS_USER=$(id -un)"
--build-arg "TRAVIS_UID=$(id -u)"
--build-arg "TRAVIS_GROUP=$(id -gn)"
--build-arg "TRAVIS_GID=$(id -g)"
build-support/docker/travis_ci_py2_ucs2/
- &docker_dry_run_release docker run --rm -t
-v "${HOME}:/travis/home"
-v "${TRAVIS_BUILD_DIR}:/travis/workdir"
travis_ci_py27_ucs2:latest
sh -c "RUN_PANTS_FROM_PEX=1 ./build-support/bin/release.sh -n"

py27_linux_build_wheels_ucs4: &py27_linux_build_wheels_ucs4
# Linux defaults to UCS4, so we simply use the pre-installed Python.
<<: *py27_linux_build_wheels_no_ucs
name: "Build Linux wheels (Py27 and UCS4 pantsbuild.pants)"
env:
- *py27_linux_test_config_env
- *base_build_wheels_env
- CACHE_NAME=linuxwheelsbuild
- CACHE_NAME=linuxwheelsbuild.ucs4
script:
- *travis_docker_image_launch
- docker run --rm -t
@@ -405,17 +434,39 @@ linux_build_wheels: &linux_build_wheels
travis_ci:latest
sh -c "RUN_PANTS_FROM_PEX=1 ./build-support/bin/release.sh -n"

osx_build_wheels: &osx_build_wheels
py27_osx_build_wheels_no_ucs: &py27_osx_build_wheels_no_ucs
<<: *py27_osx_test_config
<<: *base_build_wheels
name: "Build OSX wheels (No PEX)"
osx_image: xcode8
script:
- ./build-support/bin/release.sh -n

py27_osx_build_wheels_ucs2: &py27_osx_build_wheels_ucs2
# OSX defaults to using UCS2, so we simply use the pre-installed Python.
<<: *py27_osx_build_wheels_no_ucs
name: "Build OSX wheels (Py27 and UCS2 pantsbuild.pants)"
env:
- *py27_osx_test_config_env
- *base_build_wheels_env
- CACHE_NAME=osxwheelsbuild
script:
- ./build-support/bin/release.sh -n
- CACHE_NAME=osxwheelsbuild.ucs2

py27_osx_build_wheels_ucs4: &py27_osx_build_wheels_ucs4
<<: *py27_osx_build_wheels_no_ucs
name: "Build OSX wheels (Py27 and UCS4 pantsbuild.pants)"
env:
- *py27_osx_test_config_env
- *base_build_wheels_env
- CACHE_NAME=osxwheelsbuild.ucs4
- PYTHON_CONFIGURE_OPTS=--enable-unicode=ucs4
- PYENV_ROOT="${HOME}/.pyenv"
- PATH="${PYENV_ROOT}/shims:${PATH}"
before_install:
- curl -L https://github.com/stedolan/jq/releases/download/jq-1.5/jq-osx-amd64 -o /usr/local/bin/jq
- chmod 755 /usr/local/bin/jq
- ./build-support/bin/install_aws_cli_for_ci.sh
- git clone https://github.com/pyenv/pyenv ${PYENV_ROOT}
- ${PYENV_ROOT}/bin/pyenv install 2.7.13
- ${PYENV_ROOT}/bin/pyenv global 2.7.13

# -------------------------------------------------------------------------
# Rust tests
@@ -617,8 +668,10 @@ matrix:
- <<: *linux_rust_clippy
- <<: *cargo_audit

- <<: *linux_build_wheels
- <<: *osx_build_wheels
- <<: *py27_linux_build_wheels_ucs2
- <<: *py27_linux_build_wheels_ucs4
- <<: *py27_osx_build_wheels_ucs2
- <<: *py27_osx_build_wheels_ucs4

- <<: *py27_linux_test_config
name: "Unit tests for pants and pants-plugins (Py2.7 PEX)"
@@ -0,0 +1,35 @@
# Copyright 2017 Pants project contributors (see CONTRIBUTORS.md).
# Licensed under the Apache License, Version 2.0 (see LICENSE).

# Use our custom Centos6 image for binary compatibility with old linux distros.
FROM pantsbuild/centos6:latest

ARG PYTHON_2_VERSION=2.7.13
RUN yum install sqlite-devel -y
ENV PYENV_ROOT /pyenv-docker-build
RUN mkdir ${PYENV_ROOT}
RUN git clone https://github.com/pyenv/pyenv ${PYENV_ROOT}
ENV PYTHON_CONFIGURE_OPTS --enable-unicode=ucs2
RUN /usr/bin/scl enable devtoolset-7 -- bash -c '\
${PYENV_ROOT}/bin/pyenv install ${PYTHON_2_VERSION} \
&& ${PYENV_ROOT}/bin/pyenv global ${PYTHON_2_VERSION}'
ENV PATH "${PYENV_ROOT}/shims:${PATH}"

# Setup mount points for the travis ci user & workdir.
VOLUME /travis/home
VOLUME /travis/workdir

# Setup a non-root user to execute the build under (avoids problems with npm install).
ARG TRAVIS_USER=travis_ci
ARG TRAVIS_UID=1000
ARG TRAVIS_GROUP=root
ARG TRAVIS_GID=0

RUN groupadd --gid ${TRAVIS_GID} ${TRAVIS_GROUP} || true
RUN useradd -d /travis/home -g ${TRAVIS_GROUP} --uid ${TRAVIS_UID} ${TRAVIS_USER}
USER ${TRAVIS_USER}:${TRAVIS_GROUP}

# Our newly created user is unlikely to have a sane environment: set a locale at least.
ENV LC_ALL="en_US.UTF-8"

WORKDIR /travis/workdir
@@ -360,22 +360,51 @@ cargo_audit: &cargo_audit
# Build wheels
# -------------------------------------------------------------------------
# N.B. With Python 2, we must build pantsbuild.pants with both UCS2 and UCS4 to provide full
# compatibility for end users. This is because we constrain our ABI due to the native engine.
# See https://www.python.org/dev/peps/pep-0513/#ucs-2-vs-ucs-4-builds. Note this distinction is
# not necessary with Python 3.3+ due to flexible storage of Unicode strings (https://www.python.org/dev/peps/pep-0393/).
base_build_wheels: &base_build_wheels
stage: *test
env:
- &base_build_wheels_env RUN_PANTS_FROM_PEX=1 PREPARE_DEPLOY=1
linux_build_wheels: &linux_build_wheels
py27_linux_build_wheels_no_ucs: &py27_linux_build_wheels_no_ucs
# Similar to the bootstrap shard, we build Linux wheels in a docker image to maximize
# compatibility. This is a Py2.7 shard, so it is not subject to #6985.
<<: *travis_docker_image
<<: *py27_linux_test_config
<<: *base_build_wheels
name: "Build Linux wheels (No PEX)"
py27_linux_build_wheels_ucs2: &py27_linux_build_wheels_ucs2
<<: *py27_linux_build_wheels_no_ucs
name: "Build Linux wheels (Py27 and UCS2 pantsbuild.pants)"
env:
- *py27_linux_test_config_env
- *base_build_wheels_env
- CACHE_NAME=linuxwheelsbuild.ucs2
script:
- docker build --rm -t travis_ci_py27_ucs2
--build-arg "TRAVIS_USER=$(id -un)"
--build-arg "TRAVIS_UID=$(id -u)"
--build-arg "TRAVIS_GROUP=$(id -gn)"
--build-arg "TRAVIS_GID=$(id -g)"
build-support/docker/travis_ci_py2_ucs2/
- &docker_dry_run_release docker run --rm -t
-v "${HOME}:/travis/home"
-v "${TRAVIS_BUILD_DIR}:/travis/workdir"
travis_ci_py27_ucs2:latest
sh -c "RUN_PANTS_FROM_PEX=1 ./build-support/bin/release.sh -n"
py27_linux_build_wheels_ucs4: &py27_linux_build_wheels_ucs4
# Linux defaults to UCS4, so we simply use the pre-installed Python.
<<: *py27_linux_build_wheels_no_ucs
name: "Build Linux wheels (Py27 and UCS4 pantsbuild.pants)"
env:
- *py27_linux_test_config_env
- *base_build_wheels_env
- CACHE_NAME=linuxwheelsbuild
- CACHE_NAME=linuxwheelsbuild.ucs4
script:
- *travis_docker_image_launch
- docker run --rm -t
@@ -384,17 +413,37 @@ linux_build_wheels: &linux_build_wheels
travis_ci:latest
sh -c "RUN_PANTS_FROM_PEX=1 ./build-support/bin/release.sh -n"
osx_build_wheels: &osx_build_wheels
py27_osx_build_wheels_no_ucs: &py27_osx_build_wheels_no_ucs
<<: *py27_osx_test_config
<<: *base_build_wheels
name: "Build OSX wheels (No PEX)"
osx_image: xcode8
script:
- ./build-support/bin/release.sh -n
py27_osx_build_wheels_ucs2: &py27_osx_build_wheels_ucs2
# OSX defaults to using UCS2, so we simply use the pre-installed Python.
<<: *py27_osx_build_wheels_no_ucs
name: "Build OSX wheels (Py27 and UCS2 pantsbuild.pants)"
env:
- *py27_osx_test_config_env
- *base_build_wheels_env
- CACHE_NAME=osxwheelsbuild
script:
- ./build-support/bin/release.sh -n
- CACHE_NAME=osxwheelsbuild.ucs2
py27_osx_build_wheels_ucs4: &py27_osx_build_wheels_ucs4
<<: *py27_osx_build_wheels_no_ucs
name: "Build OSX wheels (Py27 and UCS4 pantsbuild.pants)"
env:
- *py27_osx_test_config_env
- *base_build_wheels_env
- CACHE_NAME=osxwheelsbuild.ucs4
- PYTHON_CONFIGURE_OPTS=--enable-unicode=ucs4
- PYENV_ROOT="${HOME}/.pyenv"
- PATH="${PYENV_ROOT}/shims:${PATH}"
before_install:
{{>before_install_osx}}
- git clone https://github.com/pyenv/pyenv ${PYENV_ROOT}
- ${PYENV_ROOT}/bin/pyenv install 2.7.13
- ${PYENV_ROOT}/bin/pyenv global 2.7.13
# -------------------------------------------------------------------------
# Rust tests
@@ -596,8 +645,10 @@ matrix:
- <<: *linux_rust_clippy
- <<: *cargo_audit
- <<: *linux_build_wheels
- <<: *osx_build_wheels
- <<: *py27_linux_build_wheels_ucs2
- <<: *py27_linux_build_wheels_ucs4
- <<: *py27_osx_build_wheels_ucs2
- <<: *py27_osx_build_wheels_ucs4
- <<: *py27_linux_test_config
name: "Unit tests for pants and pants-plugins (Py2.7 PEX)"

0 comments on commit 1e83f37

Please sign in to comment.
You can’t perform that action at this time.