Specify ABI for pantsbuild.pants wheel and build with both UCS2 and UCS4

.travis.yml

@@ -170,13 +170,8 @@ py36_osx_config: &py36_osx_config
       packages: &py36_osx_config_brew_packages
       - openssl
   env:
-    # Fix Python 3 issue linking to OpenSSL
     - &py36_osx_config_env >
-      PATH="/usr/local/opt/openssl/bin:$PATH"
-      LDFLAGS="-L/usr/local/opt/openssl/lib"
-      CPPFLAGS="-I/usr/local/opt/openssl/include"
-      PYENV_ROOT="${HOME}/.pyenv"
-      PATH="${PYENV_ROOT}/shims:${PATH}"
+      PATH="/usr/local/opt/openssl/bin:$PATH" LDFLAGS="-L/usr/local/opt/openssl/lib" CPPFLAGS="-I/usr/local/opt/openssl/include" PYENV_ROOT="${HOME}/.pyenv" PATH="${PYENV_ROOT}/shims:${PATH}"
   before_install:
     - curl -L https://github.com/stedolan/jq/releases/download/jq-1.5/jq-osx-amd64 -o /usr/local/bin/jq
     - chmod 755 /usr/local/bin/jq
@@ -208,11 +203,7 @@ py36_osx_test_config: &py36_osx_test_config
   env:
     # Must duplicate py36_osx_config's env because it cannot be merged into a new anchor
     - &py36_osx_test_config_env >
-      PATH="/usr/local/opt/openssl/bin:$PATH"
-      LDFLAGS="-L/usr/local/opt/openssl/lib"
-      CPPFLAGS="-I/usr/local/opt/openssl/include"
-      PYENV_ROOT="${HOME}/.pyenv"
-      PATH="${PYENV_ROOT}/shims:${PATH}"
+      PATH="/usr/local/opt/openssl/bin:$PATH" LDFLAGS="-L/usr/local/opt/openssl/lib" CPPFLAGS="-I/usr/local/opt/openssl/include" PYENV_ROOT="${HOME}/.pyenv" PATH="${PYENV_ROOT}/shims:${PATH}"
       BOOTSTRAPPED_PEX_KEY_SUFFIX=py36.osx
 
 linux_with_fuse: &linux_with_fuse
@@ -381,41 +372,113 @@ cargo_audit: &cargo_audit
 # Build wheels
 # -------------------------------------------------------------------------
 
+# N.B. With Python 2, we must build pantsbuild.pants with both UCS2 and UCS4 to provide full
+# compatibility for end users. This is because we constrain our ABI due to the native engine.
+# See https://www.python.org/dev/peps/pep-0513/#ucs-2-vs-ucs-4-builds. Note this distinction is
+# not necessary with Python 3.3+ due to flexible storage of Unicode strings (https://www.python.org/dev/peps/pep-0393/).
+#
+# We treat both Linux UCS4 and OSX UCS2 normally, as these are the defaults for those environments.
+# The Linux UCS2 and OSX UCS4 shards, however, must rebuild Python with
+# `PYTHON_CONFIGURE_OPTS=--enable-unicode=ucs{2,4}` set, along with bootstrapping Pants again rather
+# than pulling the PEX from AWS.
+
 base_build_wheels: &base_build_wheels
   stage: *test
   env:
-    - &base_build_wheels_env RUN_PANTS_FROM_PEX=1 PREPARE_DEPLOY=1
+    - &base_build_wheels_env PREPARE_DEPLOY=1
 
-linux_build_wheels: &linux_build_wheels
+py27_linux_build_wheels_no_ucs: &py27_linux_build_wheels_no_ucs
   # Similar to the bootstrap shard, we build Linux wheels in a docker image to maximize
   # compatibility. This is a Py2.7 shard, so it is not subject to #6985.
   <<: *travis_docker_image
-  <<: *py27_linux_test_config
   <<: *base_build_wheels
-  name: "Build Linux wheels (No PEX)"
+
+py27_linux_build_wheels_ucs2: &py27_linux_build_wheels_ucs2
+  <<: *py27_linux_config
+  <<: *py27_linux_build_wheels_no_ucs
+  <<: *native_engine_cache_config
+  name: "Build wheels - Linux and cp27m (UCS2)"
+  env:
+    - *base_build_wheels_env
+    - CACHE_NAME=linuxwheelsbuild.ucs2
+  script:
+    - docker build --rm -t travis_ci_py27_ucs2
+      --build-arg "TRAVIS_USER=$(id -un)"
+      --build-arg "TRAVIS_UID=$(id -u)"
+      --build-arg "TRAVIS_GROUP=$(id -gn)"
+      --build-arg "TRAVIS_GID=$(id -g)"
+      build-support/docker/travis_ci_py27_ucs2/
+    - &docker_dry_run_release docker run --rm -t
+      -v "${HOME}:/travis/home"
+      -v "${TRAVIS_BUILD_DIR}:/travis/workdir"
+      travis_ci_py27_ucs2:latest
+      sh -c "./build-support/bin/ci.sh -2b
+             && ./build-support/bin/check_pants_pex_abi.py cp27m
+             && RUN_PANTS_FROM_PEX=1 ./build-support/bin/release.sh -n"
+
+py27_linux_build_wheels_ucs4: &py27_linux_build_wheels_ucs4
+  <<: *py27_linux_build_wheels_no_ucs
+  <<: *py27_linux_test_config
+  # `py27_linux_test_config` overrides the stage set by `base_build_wheels`, so we re-override it.
+  stage: *test
+  name: "Build wheels - Linux and cp27mu (UCS4)"
   env:
     - *py27_linux_test_config_env
     - *base_build_wheels_env
-    - CACHE_NAME=linuxwheelsbuild
+    - CACHE_NAME=linuxwheelsbuild.ucs4
   script:
     - *travis_docker_image_launch
     - docker run --rm -t
       -v "${HOME}:/travis/home"
       -v "${TRAVIS_BUILD_DIR}:/travis/workdir"
       travis_ci:latest
-      sh -c "RUN_PANTS_FROM_PEX=1 ./build-support/bin/release.sh -n"
+      sh -c "./build-support/bin/check_pants_pex_abi.py cp27mu
+             && RUN_PANTS_FROM_PEX=1 ./build-support/bin/release.sh -n"
 
-osx_build_wheels: &osx_build_wheels
-  <<: *py27_osx_test_config
+py27_osx_build_wheels_no_ucs: &py27_osx_build_wheels_no_ucs
   <<: *base_build_wheels
-  name: "Build OSX wheels (No PEX)"
   osx_image: xcode8
+
+py27_osx_build_wheels_ucs2: &py27_osx_build_wheels_ucs2
+  <<: *py27_osx_test_config
+  <<: *py27_osx_build_wheels_no_ucs
+  name: "Build wheels - OSX and cp27m (UCS2)"
   env:
     - *py27_osx_test_config_env
     - *base_build_wheels_env
-    - CACHE_NAME=osxwheelsbuild
+    - CACHE_NAME=osxwheelsbuild.ucs2
+  script:
+    - ./build-support/bin/check_pants_pex_abi.py cp27m
+    - RUN_PANTS_FROM_PEX=1 ./build-support/bin/release.sh -n
+
+py27_osx_build_wheels_ucs4: &py27_osx_build_wheels_ucs4
+  <<: *py27_osx_config
+  <<: *py27_osx_build_wheels_no_ucs
+  <<: *native_engine_cache_config
+  name: "Build wheels - OSX and cp27mu (UCS4)"
+  addons:
+    brew:
+      packages:
+      - openssl
+  env:
+    - *base_build_wheels_env
+    - CACHE_NAME=osxwheelsbuild.ucs4
+    - PATH="/usr/local/opt/openssl/bin:$PATH" LDFLAGS="-L/usr/local/opt/openssl/lib" CPPFLAGS="-I/usr/local/opt/openssl/include" PYENV_ROOT="${HOME}/.pyenv" PATH="${PYENV_ROOT}/shims:${PATH}"
+
+    - PYTHON_CONFIGURE_OPTS=--enable-unicode=ucs4
+    # We set $PY to ensure the UCS4 interpreter is used when bootstrapping the PEX.
+    - PY=${PYENV_ROOT}/shims/python2.7
+  before_install:
+    - curl -L https://github.com/stedolan/jq/releases/download/jq-1.5/jq-osx-amd64 -o /usr/local/bin/jq
+    - chmod 755 /usr/local/bin/jq
+    - ./build-support/bin/install_aws_cli_for_ci.sh
+    - git clone https://github.com/pyenv/pyenv ${PYENV_ROOT}
+    - ${PYENV_ROOT}/bin/pyenv install 2.7.13
+    - ${PYENV_ROOT}/bin/pyenv global 2.7.13
   script:
-    - ./build-support/bin/release.sh -n
+    - ./build-support/bin/ci.sh -2b
+    - ./build-support/bin/check_pants_pex_abi.py cp27mu
+    - RUN_PANTS_FROM_PEX=1 ./build-support/bin/release.sh -n
 
 # -------------------------------------------------------------------------
 # Rust tests
@@ -607,7 +670,6 @@ matrix:
   include:
     - <<: *py27_linux_build_engine
     - <<: *py36_linux_build_engine
-
     - <<: *py27_osx_build_engine
     - <<: *py36_osx_build_engine
 
@@ -617,9 +679,6 @@ matrix:
     - <<: *linux_rust_clippy
     - <<: *cargo_audit
 
-    - <<: *linux_build_wheels
-    - <<: *osx_build_wheels
-
     - <<: *py27_linux_test_config
       name: "Unit tests for pants and pants-plugins (Py2.7 PEX)"
       stage: *test
@@ -637,6 +696,11 @@ matrix:
       script:
         - ./build-support/bin/travis-ci.sh -lp
 
+    - <<: *py27_linux_build_wheels_ucs2
+    - <<: *py27_linux_build_wheels_ucs4
+    - <<: *py27_osx_build_wheels_ucs2
+    - <<: *py27_osx_build_wheels_ucs4
+
     - <<: *py36_linux_test_config
       name: "Integration tests for pants - shard 0 (Py3.6 PEX)"
       env:

build-support/bin/check_pants_pex_abi.py

@@ -0,0 +1,63 @@
+#!/usr/bin/env python2.7
+# coding=utf-8
+# Copyright 2019 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+# Check that the ./pants.pex was built using the passed abi specification.
+
+from __future__ import absolute_import, division, print_function, unicode_literals
+
+import argparse
+import json
+import os.path
+import zipfile
+
+from io import open
+
+
+def main():
+  if not os.path.isfile("pants.pex"):
+    die("pants.pex not found! Ensure you are in the repository root and run " \
+        "'./build-support/bin/ci.sh -b' to bootstrap a pants.pex.")
+  expected_abi = create_parser().parse_args().abi
+  with zipfile.ZipFile("pants.pex", "r") as pex:
+    with pex.open("PEX-INFO", "r") as pex_info:
+      pex_info_content = str(pex_info.readline())
+  parsed_abis = {
+    parse_abi_from_filename(filename)
+    for filename in json.loads(pex_info_content)["distributions"].keys()
+    if parse_abi_from_filename(filename) != "none"
+  }
+  if len(parsed_abis) < 1:
+    die("No abi tag found. Expected: {}.".format(expected_abi))
+  elif len(parsed_abis) > 1:
+    die("Multiple abi tags found. Expected: {}, found: {}.".format(expected_abi, parsed_abis))
+  found_abi = list(parsed_abis)[0]
+  if found_abi != expected_abi:
+    die("pants.pex was built with the incorrect ABI. Expected: {}, found: {}.".format(expected_abi, found_abi))
+
+
+def create_parser():
+  parser = argparse.ArgumentParser(
+    description="Check that ./pants.pex was built using the passed abi specification."
+  )
+  parser.add_argument("abi", help="The expected abi, e.g. `cp27m` or `abi3`")
+  return parser
+
+
+def parse_abi_from_filename(filename):
+  """This parses out the abi from a wheel filename.
+
+  For example, `configparser-3.5.0-py2-abi3-any.whl` would return `abi3`.
+  See https://www.python.org/dev/peps/pep-0425/#use for how wheel filenames are defined."""
+  return filename.split("-")[-2]
+
+
+def die(message):
+  red = "\033[31m"
+  reset = "\033[0m"
+  raise SystemExit("{}{}{}".format(red, message, reset))
+
+
+if __name__ == "__main__":
+  main()

build-support/bin/release.sh

@@ -7,9 +7,22 @@ set -e
 ROOT=$(cd $(dirname "${BASH_SOURCE[0]}") && cd "$(git rev-parse --show-toplevel)" && pwd)
 source ${ROOT}/build-support/common.sh
 
-PY=$(which python2.7 || exit 0)
-[[ -n "${PY}" ]] || die "You must have python2.7 installed and on the path to release."
-export PY
+# Set the Python interpreter to be used for the virtualenv. Note we allow the user to
+# predefine this value so that they may point to a specific interpreter, e.g. 2.7.13 vs. 2.7.15.
+export PY="${PY:-python2.7}"
+if ! which "${PY}" >/dev/null; then
+  die "Python interpreter ${PY} not discoverable on your PATH."
+fi
+py_major_minor=$(${PY} -c 'import sys; print(".".join(map(str, sys.version_info[0:2])))')
+if [[ "${py_major_minor}" != "2.7" ]]; then
+  die "Invalid interpreter. The release script requires python2.7, and you are using python${py_major_minor}."
+fi
+
+# Also set PANTS_PYTHON_SETUP_INTERPRETER_CONSTRAINTS. We set this to the exact Python version
+# to resolve any potential ambiguity when multiple Python interpreters are discoverable, such as
+# Python 2.7.13 vs. 2.7.15.
+py_major_minor_patch=$(${PY} -c 'import sys; print(".".join(map(str, sys.version_info[0:3])))')
+export PANTS_PYTHON_SETUP_INTERPRETER_CONSTRAINTS="${PANTS_PYTHON_SETUP_INTERPRETER_CONSTRAINTS:-['CPython==${py_major_minor_patch}']}"
 
 function run_local_pants() {
   ${ROOT}/pants "$@"

build-support/docker/travis_ci_py27_ucs2/Dockerfile

@@ -0,0 +1,49 @@
+# Copyright 2019 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+# NB: this file duplicates travis_ci/Dockerfile, except it installs Python 2.7 with UCS2.
+# We do not include this change directly in centos6/Dockerfile nor in travis_ci/Dockerfile
+# because we only want access to this Python interpreter in the Build Wheels Linux UCS2 shard,
+# so it is not helpful to other shards. Rather, it would make those shards more finicky to deal
+# with by resulting in two Python 2.7 installs: system vs. pyenv.
+
+# Use our custom Centos6 image for binary compatibility with old linux distros.
+FROM pantsbuild/centos6:latest
+
+# Note we use 2.7.15, rather than 2.7.13, as the centos6 image already comes with 2.7.13
+# installed, which uses UCS4 instead of UCS2. This allows us to disambiguate which Python 2
+# interpreter to use when `ci.sh` and `release.sh` set the interpreter constraints for
+# Pants, and thus for the built ./pants.pex. We set $PY to the exact Python 2.7 version we want
+# to ensure the PEX is bootstrapped with UCS 2.
+ARG PYTHON_2_VERSION=2.7.15
+# TODO(7064): remove this yum install line once we update the base Centos6 image to include this dependency.
+RUN yum install sqlite-devel -y
+ENV PYENV_ROOT /pyenv-docker-build
+RUN mkdir ${PYENV_ROOT}
+RUN git clone https://github.com/pyenv/pyenv ${PYENV_ROOT}
+ENV PYTHON_CONFIGURE_OPTS --enable-unicode=ucs2
+RUN /usr/bin/scl enable devtoolset-7 -- bash -c '\
+    ${PYENV_ROOT}/bin/pyenv install ${PYTHON_2_VERSION} \
+    && ${PYENV_ROOT}/bin/pyenv global ${PYTHON_2_VERSION}'
+ENV PATH "${PYENV_ROOT}/shims:${PATH}"
+ENV PY "${PYENV_ROOT}/shims/python2.7"
+ENV PEX_PYTHON_PATH "${PYENV_ROOT}/shims/python2.7"
+
+# Setup mount points for the travis ci user & workdir.
+VOLUME /travis/home
+VOLUME /travis/workdir
+
+# Setup a non-root user to execute the build under (avoids problems with npm install).
+ARG TRAVIS_USER=travis_ci
+ARG TRAVIS_UID=1000
+ARG TRAVIS_GROUP=root
+ARG TRAVIS_GID=0
+
+RUN groupadd --gid ${TRAVIS_GID} ${TRAVIS_GROUP} || true
+RUN useradd -d /travis/home -g ${TRAVIS_GROUP} --uid ${TRAVIS_UID} ${TRAVIS_USER}
+USER ${TRAVIS_USER}:${TRAVIS_GROUP}
+
+# Our newly created user is unlikely to have a sane environment: set a locale at least.
+ENV LC_ALL="en_US.UTF-8"
+
+WORKDIR /travis/workdir

build-support/travis/env_osx_with_pyenv.mustache

@@ -0,0 +1 @@
+PATH="/usr/local/opt/openssl/bin:$PATH" LDFLAGS="-L/usr/local/opt/openssl/lib" CPPFLAGS="-I/usr/local/opt/openssl/include" PYENV_ROOT="${HOME}/.pyenv" PATH="${PYENV_ROOT}/shims:${PATH}"

build-support/travis/generate_travis_yml.py

@@ -25,12 +25,14 @@
 
 def generate_travis_yml():
   """Generates content for a .travis.yml file from templates."""
-  template = pkg_resources.resource_string(
-    __name__, 'travis.yml.mustache').decode('utf-8')
-  before_install_linux = pkg_resources.resource_string(
-    __name__, 'before_install_linux.mustache').decode('utf-8')
-  before_install_osx = pkg_resources.resource_string(
-    __name__, 'before_install_osx.mustache').decode('utf-8')
+  def get_mustache_file(file_name):
+    return pkg_resources.resource_string(__name__, file_name).decode('utf-8')
+
+  template = get_mustache_file('travis.yml.mustache')
+  before_install_linux = get_mustache_file('before_install_linux.mustache')
+  before_install_osx = get_mustache_file('before_install_osx.mustache')
+  env_osx_with_pyenv = get_mustache_file('env_osx_with_pyenv.mustache')
+
   context = {
     'header': HEADER,
     'py3_integration_shards': range(0, num_py3_integration_shards),
@@ -42,6 +44,7 @@ def generate_travis_yml():
   }
   renderer = pystache.Renderer(partials={
     'before_install_linux': before_install_linux,
-    'before_install_osx': before_install_osx
+    'before_install_osx': before_install_osx,
+    'env_osx_with_pyenv': env_osx_with_pyenv
   })
   print(renderer.render(template, context))