Modernize virtualenv.

.gitignore

@@ -37,9 +37,9 @@ out/
 # remove root-level /bin, which is created by Eclipse
 /bin/
 /build-support/*.pex
+/build-support/pex/
 /build-support/virtualenvs
-/build-support/virtualenv-*
-/build-support/virtualenv.dist/
+/build-support/virtualenv/
 /build-support/phab/
 /build-support/twine-deps.venv/
 /dist/

build-support/bin/check_shell.sh

@@ -17,8 +17,8 @@ if [[ -n "${bad_output}" ]]; then
   exit_code=1
 fi
 
-# Skip release.sh because it has a custom function for failing noisily.
-bad_output="$(find ./* -name '*.sh' -not -path './build-support/bin/release.sh' -print0 \
+# Skip common.sh because it has a custom function for failing noisily.
+bad_output="$(find ./* -name '*.sh' -not -path './build-support/common.sh' -print0 \
   | xargs -0 grep -H 'curl '| grep -v 'curl --fail' | cut -d: -f1 \
   | grep -v build-support/bin/check_shell.sh \
   || :)"

build-support/bin/release.sh

@@ -6,18 +6,6 @@ set -e
 
 ROOT=$(cd "$(dirname "${BASH_SOURCE[0]}")" && cd "$(git rev-parse --show-toplevel)" && pwd)
 
-function safe_curl() {
-  real_curl="$(command -v curl)"
-  set +e
-  "${real_curl}" --fail -SL "$@"
-  exit_code=$?
-  set -e
-  if [[ "${exit_code}" -ne 0 ]]; then
-    echo >&2 "Curl failed with args: $*"
-    exit 1
-  fi
-}
-
 # shellcheck source=build-support/common.sh
 source "${ROOT}/build-support/common.sh"
 
@@ -69,29 +57,6 @@ readonly DEPLOY_PANTS_WHEEL_DIR="${DEPLOY_DIR}/${DEPLOY_PANTS_WHEELS_PATH}"
 # only pants core is included.
 : "${PANTS_PEX_PACKAGES:="pantsbuild.pants"}"
 
-# URL from which pex release binaries can be downloaded.
-: "${PEX_DOWNLOAD_PREFIX:="https://github.com/pantsbuild/pex/releases/download"}"
-
-function requirement() {
-  package="$1"
-  grep "^${package}[^A-Za-z0-9]" "${ROOT}/3rdparty/python/requirements.txt" || die "Could not find requirement for ${package}"
-}
-
-function run_pex() {
-  # TODO: Cache this in case we run pex multiple times
-  (
-    PEX_VERSION="$(requirement pex | sed -e "s|pex==||")"
-
-    pexdir="$(mktemp -d -t build_pex.XXXXX)"
-    trap 'rm -rf "${pexdir}"' EXIT
-
-    pex="${pexdir}/pex"
-
-    safe_curl -s "${PEX_DOWNLOAD_PREFIX}/v${PEX_VERSION}/pex" > "${pex}"
-    "${PY}" "${pex}" "$@"
-  )
-}
-
 function run_packages_script() {
   (
     cd "${ROOT}"
@@ -177,7 +142,7 @@ function activate_tmp_venv() {
   # place, Shellcheck will not be able to find it so we tell Shellcheck to ignore the file.
   # shellcheck source=/dev/null
   VENV_DIR=$(mktemp -d -t pants.XXXXX) && \
-  "${ROOT}/build-support/virtualenv" "$VENV_DIR" && \
+  run_virtualenv "$VENV_DIR" && \
   source "$VENV_DIR/bin/activate"
 }
 
@@ -300,7 +265,7 @@ function activate_twine() {
   local -r venv_dir="${ROOT}/build-support/twine-deps.venv"
 
   rm -rf "${venv_dir}"
-  "${ROOT}/build-support/virtualenv" "${venv_dir}"
+  run_virtualenv "${venv_dir}"
   # Because the venv/bin/activate script's location is dynamic and not located in a fixed
   # place, Shellcheck will not be able to find it so we tell Shellcheck to ignore the file.
   # shellcheck source=/dev/null
@@ -310,12 +275,13 @@ function activate_twine() {
 
 function execute_pex() {
   run_pex \
-      --no-build \
-      --no-pypi \
-      --disable-cache \
-      -f "${DEPLOY_PANTS_WHEEL_DIR}/${PANTS_UNSTABLE_VERSION}" \
-      -f "${DEPLOY_3RDPARTY_WHEEL_DIR}/${PANTS_UNSTABLE_VERSION}" \
-      "$@"
+    --python "${PY}" \
+    --no-build \
+    --no-pypi \
+    --disable-cache \
+    -f "${DEPLOY_PANTS_WHEEL_DIR}/${PANTS_UNSTABLE_VERSION}" \
+    -f "${DEPLOY_3RDPARTY_WHEEL_DIR}/${PANTS_UNSTABLE_VERSION}" \
+    "$@"
 }
 
 function build_pex() {

build-support/bin/shellcheck.py

@@ -27,12 +27,10 @@ def run_shellcheck() -> None:
     targets = set(glob("./**/*.sh", recursive=True)) | {
         "./pants",
         "./build-support/pants_venv",
-        "./build-support/virtualenv",
         "./build-support/githooks/pre-commit",
         "./build-support/githooks/prepare-commit-msg",
     }
     targets -= set(glob("./build-support/bin/native/src/**/*.sh", recursive=True))
-    targets -= set(glob("./build-support/virtualenv.dist/**/*.sh", recursive=True))
     targets -= set(glob("./build-support/twine-deps.venv/**/*.sh", recursive=True))
     command = ["shellcheck", "--shell=bash", "--external-sources"] + sorted(targets)
     try:

build-support/common.sh

@@ -68,3 +68,75 @@ function git_merge_base() {
   # This prints the tracking branch if set and otherwise falls back to local "master".
   git rev-parse --symbolic-full-name --abbrev-ref HEAD@\{upstream\} 2>/dev/null || echo 'master'
 }
+
+function safe_curl() {
+  real_curl="$(command -v curl)"
+  set +e
+  "${real_curl}" --fail -SL "$@"
+  exit_code=$?
+  set -e
+  if [[ "${exit_code}" -ne 0 ]]; then
+    echo >&2 "Curl failed with args: $*"
+    exit 1
+  fi
+}
+
+function root_dir() {
+  cd "$(dirname "${BASH_SOURCE[0]}")" && cd "$(git rev-parse --show-toplevel)" && pwd
+}
+
+function requirement() {
+  package="$1"
+  grep "^${package}[^A-Za-z0-9]" "$(root_dir)/3rdparty/python/requirements.txt" || \
+    die "Could not find requirement for ${package}"
+}
+
+# URL from which pex release binaries can be downloaded.
+: "${PEX_DOWNLOAD_PREFIX:="https://github.com/pantsbuild/pex/releases/download"}"
+
+function run_pex() {
+  (
+    PEX_VERSION="$(requirement pex | sed -e "s|pex==||")"
+    PEX_DIR="$(root_dir)/build-support/pex/${PEX_VERSION}"
+    if ! [ -x "${PEX_DIR}/pex" ]; then
+      pex_url="${PEX_DOWNLOAD_PREFIX}/v${PEX_VERSION}/pex"
+      log "Downloading pex==${PEX_VERSION} from ${pex_url}..."
+
+      pexdir="$(mktemp -d -t download_pex.XXXXX)"
+      trap 'rm -rf "${pexdir}"' EXIT
+
+      safe_curl -s "${pex_url}" > "${pexdir}/pex" && \
+      chmod +x "${pexdir}/pex" && \
+      rm -rf "${PEX_DIR}" && \
+      mkdir -p "$(dirname "${PEX_DIR}")" && \
+      mv "${pexdir}" "${PEX_DIR}"
+    fi
+
+    "${PEX_DIR}/pex" "$@"
+  )
+}
+
+
+: "${VIRTUALENV_VERSION:="20.0.26"}"
+
+function run_virtualenv() {
+  (
+    VIRTUALENV_DIR="$(root_dir)/build-support/virtualenv/${VIRTUALENV_VERSION}"
+    if ! [ -x "${VIRTUALENV_DIR}/virtualenv.pex" ]; then
+      log "Building PEX for virtualenv==${VIRTUALENV_VERSION}..."
+      pexdir="$(mktemp -d -t virtualenv_pex.XXXXX)"
+      trap 'rm -rf "${pexdir}"' EXIT
+
+      run_pex \
+        --python="${PY}" \
+        "virtualenv==${VIRTUALENV_VERSION}" \
+        -c virtualenv \
+        -o "${pexdir}/virtualenv.pex" && \
+      rm -rf "${VIRTUALENV_DIR}" && \
+      mkdir -p "$(dirname "${VIRTUALENV_DIR}")" && \
+      mv "${pexdir}" "${VIRTUALENV_DIR}"
+    fi
+
+    "${VIRTUALENV_DIR}/virtualenv.pex" "$@"
+  )
+}

build-support/pants_venv

@@ -33,7 +33,7 @@ function remove_venv() {
 
 function create_venv() {
   remove_venv
-  "${REPO_ROOT}/build-support/virtualenv" "$(venv_dir)"
+  run_virtualenv "$(venv_dir)"
 }
 
 function ensure_gcc() {