Install Python 2.7 and 3.6 on Centos6 base image through Pyenv

.travis.yml

@@ -344,9 +344,10 @@ py27_linux_build_engine: &py27_linux_build_engine
   name: "Build Linux native engine and pants.pex (Py2.7 PEX)"
   env:
     - docker_image_name=travis_ci
-    # We also build fs_util.
-    - docker_run_command="./build-support/bin/ci.sh -2b
-                          && ./build-support/bin/release.sh -f"
+    # TODO: While this image shouldn't have any cache to fetch anything from, it fails to find
+    # libpython2.7.so.1 during the cargo build, despite the image being built with --enable-shared,
+    # unless the -x argument is added. This isn't expected to affect build time too much.
+    - docker_run_command="./build-support/bin/ci.sh -2bx && ./build-support/bin/release.sh -f"
     - PREPARE_DEPLOY=1
     - CACHE_NAME=linuxpexbuild.py27
     - BOOTSTRAPPED_PEX_KEY_SUFFIX=py27.linux
@@ -356,7 +357,7 @@ py36_linux_build_engine: &py36_linux_build_engine
   <<: *base_linux_build_engine
   name: "Build Linux native engine and pants.pex (Py3.6 PEX)"
   env:
-    - docker_image_name=travis_ci_py36
+    - docker_image_name=travis_ci
     - docker_run_command="./build-support/bin/ci.sh -b"
     - CACHE_NAME=linuxpexbuild.py36
     - BOOTSTRAPPED_PEX_KEY_SUFFIX=py36.linux
@@ -563,7 +564,7 @@ py36_linux_build_wheels: &py36_linux_build_wheels
   env:
     - *py36_linux_test_config_env
     - *base_build_wheels_env
-    - docker_image_name=travis_ci_py36
+    - docker_image_name=travis_ci
     - docker_run_command="./build-support/bin/check_pants_pex_abi.py cp36m
                           && RUN_PANTS_FROM_PEX=1 ./build-support/bin/release.sh -3n"
     - CACHE_NAME=linuxwheelsbuild.abi3

build-support/docker/centos6/Dockerfile

@@ -4,18 +4,38 @@
 # Use centos6 for compatibility with glibc 2.12.
 FROM centos:6
 
-# Install python 2.7.13, (more) modern gcc, and a JDK.
+# Install a (more) modern gcc, a JDK, and dependencies for installing Python through Pyenv.
+# Also install sqlite-devel because Python 3 errors on startup if not -- see
+# https://stackoverflow.com/a/39907500/2518889.
 RUN yum -y update
+# TODO: figure out why this needs to be installed first for /usr/bin/scl to work!
 RUN yum install -y centos-release-scl
 RUN yum install -y \
-  devtoolset-7-gcc \
-  devtoolset-7-gcc-c++ \
-  git \
-  java-1.8.0-openjdk-devel \
-  python27 \
-  libffi \
-  libffi-devel \
-  openssl-devel
+        bzip2-devel \
+        devtoolset-7-gcc{,-c++} \
+        git \
+        java-1.8.0-openjdk-devel \
+        libffi-devel \
+        openssl-devel \
+        readline-devel \
+        sqlite-devel \
+        zlib-devel
 
-# By default, execute in an environment with python27 enabled.
-ENTRYPOINT ["/usr/bin/scl", "enable", "python27", "devtoolset-7",  "--"]
+ARG PYTHON_27_VERSION=2.7.13
+ARG PYTHON_36_VERSION=3.6.8
+# NB: PYENV_ROOT must be set for `pyenv install` to be available. This failure mode is not mentioned
+# in https://github.com/pyenv/pyenv#basic-github-checkout.
+ENV PYENV_ROOT /pyenv-docker-build
+ENV PYENV_BIN "${PYENV_ROOT}/bin/pyenv"
+RUN git clone https://github.com/pyenv/pyenv ${PYENV_ROOT}
+
+# Install Python 2.7 and 3.6.
+# Build the Python shared library, as we use it to build the engine.
+ENV PYTHON_CONFIGURE_OPTS="--enable-shared"
+RUN /usr/bin/scl enable devtoolset-7 -- ${PYENV_BIN} install ${PYTHON_27_VERSION}
+RUN /usr/bin/scl enable devtoolset-7 -- ${PYENV_BIN} install ${PYTHON_36_VERSION}
+RUN ${PYENV_BIN} global ${PYTHON_27_VERSION} ${PYTHON_36_VERSION}
+ENV PATH "${PYENV_ROOT}/shims:${PATH}"
+
+# Expose the installed gcc to the invoking shell.
+ENTRYPOINT ["/usr/bin/scl", "enable", "devtoolset-7",  "--"]

build-support/docker/travis_ci/Dockerfile

@@ -2,11 +2,53 @@
 # Licensed under the Apache License, Version 2.0 (see LICENSE).
 
 # Use our custom Centos6 image for binary compatibility with old linux distros.
-FROM pantsbuild/centos6:latest
+ARG BASE_IMAGE=pantsbuild/centos6:latest
+FROM ${BASE_IMAGE}
 
-# Setup mount points for the travis ci user & workdir.
-VOLUME /travis/home
-VOLUME /travis/workdir
+# TODO(7064) These changes belong in centos6/Dockerfile, because we always want that image to
+# have Py3 available and also avoid the cost of rebuilding Python 3 every time in CI. However,
+# this requires publishing an updated image to Docker hub, so we copy the code here until then.
+
+### BEGIN CENTOS6 DOCKERFILE CHANGES
+RUN yum -y update
+# TODO: figure out why this needs to be installed first for /usr/bin/scl to work!
+RUN yum install -y centos-release-scl
+RUN yum install -y \
+        bzip2-devel \
+        devtoolset-7-gcc{,-c++} \
+        git \
+        java-1.8.0-openjdk-devel \
+        libffi-devel \
+        openssl-devel \
+        readline-devel \
+        sqlite-devel \
+        zlib-devel
+
+ARG PYTHON_27_VERSION=2.7.13
+ARG PYTHON_36_VERSION=3.6.8
+# NB: PYENV_ROOT must be set for `pyenv install` to be available. This failure mode is not mentioned
+# in https://github.com/pyenv/pyenv#basic-github-checkout.
+ENV PYENV_ROOT "${PYENV_ROOT:-/pyenv-docker-build}"
+ENV PYENV_BIN "${PYENV_ROOT}/bin/pyenv"
+ENV PYTHON_CONFIGURE_OPTS="--enable-shared"
+RUN if [[ ! -d "${PYENV_ROOT}" ]]; then \
+  git clone https://github.com/pyenv/pyenv ${PYENV_ROOT} \
+  && /usr/bin/scl enable devtoolset-7 -- ${PYENV_BIN} install ${PYTHON_27_VERSION} \
+  && /usr/bin/scl enable devtoolset-7 -- ${PYENV_BIN} install ${PYTHON_36_VERSION} \
+  && ${PYENV_BIN} global ${PYTHON_27_VERSION} ${PYTHON_36_VERSION}; \
+  fi
+ENV PATH "${PYENV_ROOT}/shims:${PATH}"
+
+# Expose the installed gcc to the invoking shell.
+ENTRYPOINT ["/usr/bin/scl", "enable", "devtoolset-7",  "--"]
+
+### END CENTOS6 DOCKERFILE CHANGES
+
+ARG TRAVIS_HOME_DIR_PATH=/travis/home
+ARG TRAVIS_WORK_DIR_PATH=/travis/workdir
+
+RUN mkdir -p ${TRAVIS_HOME_DIR_PATH}
+RUN mkdir -p ${TRAVIS_WORK_DIR_PATH}
 
 # Setup a non-root user to execute the build under (avoids problems with npm install).
 ARG TRAVIS_USER=travis_ci
@@ -15,7 +57,11 @@ ARG TRAVIS_GROUP=root
 ARG TRAVIS_GID=0
 
 RUN groupadd --gid ${TRAVIS_GID} ${TRAVIS_GROUP} || true
-RUN useradd -d /travis/home -g ${TRAVIS_GROUP} --uid ${TRAVIS_UID} ${TRAVIS_USER}
+RUN useradd -d ${TRAVIS_HOME_DIR_PATH} -g ${TRAVIS_GROUP} --uid ${TRAVIS_UID} ${TRAVIS_USER}
+
+RUN chown -R ${TRAVIS_USER} ${TRAVIS_HOME_DIR_PATH}
+RUN chown -R ${TRAVIS_USER} ${TRAVIS_WORK_DIR_PATH}
+
 USER ${TRAVIS_USER}:${TRAVIS_GROUP}
 
 # Our newly created user is unlikely to have a sane environment: set a locale at least.
@@ -26,4 +72,7 @@ ENV LC_ALL="en_US.UTF-8"
 # clean-all without this env var.
 ENV ONLY_USING_SINGLE_PYTHON_VERSION 'true'
 
-WORKDIR /travis/workdir
+# Expose the installed gcc to the invoking shell.
+ENTRYPOINT ["/usr/bin/scl", "enable", "devtoolset-7",  "--"]
+
+WORKDIR ${TRAVIS_WORK_DIR_PATH}

build-support/docker/travis_ci_py27_ucs2/Dockerfile

@@ -21,10 +21,12 @@ RUN yum install sqlite-devel -y
 ENV PYENV_ROOT "${PYENV_ROOT:-/pyenv-docker-build}"
 ENV PYENV_BIN "${PYENV_ROOT}/bin/pyenv"
 RUN if [[ ! -d "${PYENV_ROOT}" ]]; then git clone https://github.com/pyenv/pyenv ${PYENV_ROOT}; fi
+
 ENV PYTHON_CONFIGURE_OPTS --enable-unicode=ucs2
-RUN /usr/bin/scl enable devtoolset-7 -- bash -c '${PYENV_BIN} install ${PYTHON_27_VERSION_UCS2}'
-RUN /usr/bin/scl enable devtoolset-7 -- bash -c '${PYENV_BIN} global ${PYTHON_27_VERSION_UCS2}'
+RUN /usr/bin/scl enable devtoolset-7 -- ${PYENV_BIN} install ${PYTHON_27_VERSION_UCS2}
+RUN /usr/bin/scl enable devtoolset-7 -- ${PYENV_BIN} global ${PYTHON_27_VERSION_UCS2}
 ENV PATH "${PYENV_ROOT}/shims:${PATH}"
+
 ENV PY "${PYENV_ROOT}/shims/python2.7"
 ENV PEX_PYTHON_PATH "${PYENV_ROOT}/shims/python2.7"
 

build-support/travis/travis.yml.mustache

@@ -302,9 +302,10 @@ py27_linux_build_engine: &py27_linux_build_engine
   name: "Build Linux native engine and pants.pex (Py2.7 PEX)"
   env:
     - docker_image_name=travis_ci
-    # We also build fs_util.
-    - docker_run_command="./build-support/bin/ci.sh -2b
-                          && ./build-support/bin/release.sh -f"
+    # TODO: While this image shouldn't have any cache to fetch anything from, it fails to find
+    # libpython2.7.so.1 during the cargo build, despite the image being built with --enable-shared,
+    # unless the -x argument is added. This isn't expected to affect build time too much.
+    - docker_run_command="./build-support/bin/ci.sh -2bx && ./build-support/bin/release.sh -f"
     - PREPARE_DEPLOY=1
     - CACHE_NAME=linuxpexbuild.py27
     - BOOTSTRAPPED_PEX_KEY_SUFFIX=py27.linux
@@ -314,7 +315,7 @@ py36_linux_build_engine: &py36_linux_build_engine
   <<: *base_linux_build_engine
   name: "Build Linux native engine and pants.pex (Py3.6 PEX)"
   env:
-    - docker_image_name=travis_ci_py36
+    - docker_image_name=travis_ci
     - docker_run_command="./build-support/bin/ci.sh -b"
     - CACHE_NAME=linuxpexbuild.py36
     - BOOTSTRAPPED_PEX_KEY_SUFFIX=py36.linux
@@ -510,7 +511,7 @@ py36_linux_build_wheels: &py36_linux_build_wheels
   env:
     - *py36_linux_test_config_env
     - *base_build_wheels_env
-    - docker_image_name=travis_ci_py36
+    - docker_image_name=travis_ci
     - docker_run_command="./build-support/bin/check_pants_pex_abi.py cp36m
                           && RUN_PANTS_FROM_PEX=1 ./build-support/bin/release.sh -3n"
     - CACHE_NAME=linuxwheelsbuild.abi3