Upgrade to pex 1.4.5. (pantsbuild#6267)

Eliminate a few workarounds that are no longer needed and
switch to using the released pex.pex in the release script.

Work towards pantsbuild#5922

3rdparty/python/requirements.txt

@@ -14,7 +14,7 @@ mock==2.0.0
 packaging==16.8
 parameterized==0.6.1
 pathspec==0.5.0
-pex==1.4.3
+pex==1.4.5
 psutil==4.3.0
 pycodestyle==2.4.0
 pyflakes==2.0.0

build-support/bin/release.sh

@@ -584,8 +584,6 @@ function build_pex() {
     build_3rdparty_packages "${PANTS_UNSTABLE_VERSION}"
   fi
 
-  activate_tmp_venv && trap deactivate RETURN && pip install "pex==1.4.3" || die "Failed to install pex."
-
   local requirements=()
   for pkg_name in $PANTS_PEX_PACKAGES; do
     requirements=("${requirements[@]}" "${pkg_name}==${PANTS_UNSTABLE_VERSION}")
@@ -596,16 +594,27 @@ function build_pex() {
     platform_flags=("${platform_flags[@]}" "--platform=${platform}")
   done
 
-  pex \
-    -o "${dest}" \
-    --entry-point="pants.bin.pants_loader:main" \
-    --no-build \
-    --no-pypi \
-    --disable-cache \
-    "${platform_flags[@]}" \
-    -f "${DEPLOY_PANTS_WHEEL_DIR}/${PANTS_UNSTABLE_VERSION}" \
-    -f "${DEPLOY_3RDPARTY_WHEEL_DIR}/${PANTS_UNSTABLE_VERSION}" \
-    "${requirements[@]}"
+  (
+    PEX_VERSION=$(grep "pex==" "${ROOT}/3rdparty/python/requirements.txt" | sed -e "s|pex==||")
+    PEX_PEX=pex27
+
+    cd $(mktemp -d -t build_pex.XXXXX)
+    trap "rm -rf $(pwd -P)" EXIT
+
+    curl -sSL https://github.com/pantsbuild/pex/releases/download/v${PEX_VERSION}/${PEX_PEX} -O
+    chmod +x ./${PEX_PEX}
+
+    ./${PEX_PEX} \
+      -o "${dest}" \
+      -c pants \
+      --no-build \
+      --no-pypi \
+      --disable-cache \
+      "${platform_flags[@]}" \
+      -f "${DEPLOY_PANTS_WHEEL_DIR}/${PANTS_UNSTABLE_VERSION}" \
+      -f "${DEPLOY_3RDPARTY_WHEEL_DIR}/${PANTS_UNSTABLE_VERSION}" \
+      "${requirements[@]}"
+  )
 
   if [[ "${PANTS_PEX_RELEASE}" == "stable" ]]; then
     mkdir -p "$(dirname "${stable_dest}")"

src/python/pants/backend/native/subsystems/conan.py

@@ -13,8 +13,9 @@
 from pex.pex_info import PexInfo
 
 from pants.backend.python.python_requirement import PythonRequirement
+from pants.backend.python.subsystems.python_repos import PythonRepos
+from pants.backend.python.subsystems.python_setup import PythonSetup
 from pants.backend.python.tasks.pex_build_util import dump_requirements
-from pants.backend.python.tasks.wrapped_pex import WrappedPEX
 from pants.base.build_environment import get_pants_cachedir
 from pants.subsystem.subsystem import Subsystem
 from pants.util.dirutil import safe_concurrent_creation
@@ -49,6 +50,10 @@ class Conan(Subsystem):
   def implementation_version(cls):
     return super(Conan, cls).implementation_version() + [('Conan', 0)]
 
+  @classmethod
+  def subsystem_dependencies(cls):
+    return super(Conan, cls).subsystem_dependencies() + (PythonRepos, PythonSetup)
+
   @classmethod
   def register_options(cls, register):
     super(Conan, cls).register_options(register)
@@ -57,22 +62,18 @@ def register_options(cls, register):
 
   class ConanBinary(datatype(['pex'])):
     """A `conan` PEX binary."""
-    pass
 
   def bootstrap_conan(self):
     pex_info = PexInfo.default()
     pex_info.entry_point = 'conans.conan'
     conan_bootstrap_dir = os.path.join(get_pants_cachedir(), 'conan_support')
     conan_pex_path = os.path.join(conan_bootstrap_dir, 'conan_binary')
     interpreter = PythonInterpreter.get()
-    if os.path.exists(conan_pex_path):
-      conan_binary = WrappedPEX(PEX(conan_pex_path, interpreter))
-      return self.ConanBinary(pex=conan_binary)
-    else:
+    if not os.path.exists(conan_pex_path):
       with safe_concurrent_creation(conan_pex_path) as safe_path:
         builder = PEXBuilder(safe_path, interpreter, pex_info=pex_info)
         reqs = [PythonRequirement(req) for req in self.get_options().conan_requirements]
         dump_requirements(builder, interpreter, reqs, logger)
         builder.freeze()
-      conan_binary = WrappedPEX(PEX(conan_pex_path, interpreter))
-      return self.ConanBinary(pex=conan_binary)
+    conan_binary = PEX(conan_pex_path, interpreter)
+    return self.ConanBinary(pex=conan_binary)

src/python/pants/backend/native/tasks/native_external_library_fetch.py

@@ -179,9 +179,7 @@ def ensure_conan_remote_configuration(self, conan_binary):
     # and replace it with Pants-controlled remotes.
     remove_conan_center_remote_cmdline = self._remove_conan_center_remote_cmdline(conan_binary)
     try:
-      # Slice the command line because subprocess errors when the first element in the
-      # list of command strings is the setting of an environment variable.
-      stdout = subprocess.check_output(remove_conan_center_remote_cmdline.split()[1:])
+      stdout = subprocess.check_output(remove_conan_center_remote_cmdline)
       self.context.log.debug(stdout)
     except subprocess.CalledProcessError as e:
       if not "'conan-center' not found in remotes" in e.output:
@@ -197,7 +195,7 @@ def ensure_conan_remote_configuration(self, conan_binary):
                                                                             index_num,
                                                                             remote_url)
       try:
-        stdout = subprocess.check_output(add_pants_conan_remote_cmdline.split()[1:])
+        stdout = subprocess.check_output(add_pants_conan_remote_cmdline)
         self.context.log.debug(stdout)
       except subprocess.CalledProcessError as e:
         if not "already exists in remotes" in e.output:
@@ -252,7 +250,7 @@ def _fetch_packages(self, vt, vts_results_dir):
 
         # Invoke conan to pull package from remote.
         try:
-          stdout = subprocess.check_output(cmdline.split()[1:])
+          stdout = subprocess.check_output(cmdline)
         except subprocess.CalledProcessError as e:
           raise self.NativeExternalLibraryFetchError(
             "Error invoking conan for fetch task: {}\n".format(e.output)

src/python/pants/backend/python/interpreter_cache.py

@@ -13,7 +13,7 @@
 from pex.resolver import resolve
 from pex.variables import Variables
 
-from pants.backend.python.pex_util import create_bare_interpreter, expand_and_maybe_adjust_platform
+from pants.backend.python.pex_util import expand_and_maybe_adjust_platform
 from pants.backend.python.targets.python_target import PythonTarget
 from pants.base.exceptions import TaskError
 from pants.process.lock import OwnerPrintingInterProcessFileLock
@@ -105,7 +105,7 @@ def _interpreter_from_path(self, path, filters):
       executable = os.readlink(os.path.join(path, 'python'))
     except OSError:
       return None
-    interpreter = create_bare_interpreter(executable)
+    interpreter = PythonInterpreter.from_binary(executable, include_site_extras=False)
     if self._matches(interpreter, filters):
       return self._resolve(interpreter)
     return None

src/python/pants/backend/python/pex_util.py

@@ -13,20 +13,6 @@
 logger = logging.getLogger(__name__)
 
 
-def create_bare_interpreter(binary_path):
-  """Creates an interpreter for python binary at the given path.
-
-  The interpreter is bare in that it has no extras associated with it.
-
-  :returns: A bare python interpreter with no extras.
-  :rtype: :class:`pex.interpreter.PythonInterpreter`
-  """
-  # TODO(John Sirois): Replace with a more direct PythonInterpreter construction API call when
-  # https://github.com/pantsbuild/pex/issues/510 is fixed.
-  interpreter_with_extras = PythonInterpreter.from_binary(binary_path)
-  return PythonInterpreter(binary_path, interpreter_with_extras.identity, extras=None)
-
-
 def _interpreter_str(interp):
   ident = interp.identity
   return ('PythonInterpreter({binary!r}, {identity!r} with extended info: '

src/python/pants/backend/python/tasks/select_interpreter.py

@@ -10,7 +10,6 @@
 from pex.interpreter import PythonInterpreter
 
 from pants.backend.python.interpreter_cache import PythonInterpreterCache
-from pants.backend.python.pex_util import create_bare_interpreter
 from pants.backend.python.subsystems.python_repos import PythonRepos
 from pants.backend.python.subsystems.python_setup import PythonSetup
 from pants.backend.python.targets.python_requirement_library import PythonRequirementLibrary
@@ -102,7 +101,7 @@ def _get_interpreter(interpreter_path_file):
     with open(interpreter_path_file, 'r') as infile:
       lines = infile.readlines()
       binary = lines[0].strip()
-      interpreter = create_bare_interpreter(binary)
+      interpreter = PythonInterpreter.from_binary(binary, include_site_extras=False)
       for line in lines[1:]:
         dist_name, dist_version, location = line.strip().split('\t')
         interpreter = interpreter.with_extra(dist_name, dist_version, location)

src/python/pants/backend/python/tasks/wrapped_pex.py

@@ -19,7 +19,6 @@ class WrappedPEX(object):
   """
 
   _PEX_PATH_ENV_VAR_NAME = 'PEX_PATH'
-  _PEX_PYTHON_PATH_ENV_VAR_NAME = 'PEX_PYTHON_PATH'
 
   def __init__(self, pex, extra_pex_paths=None):
     """
@@ -38,28 +37,17 @@ def path(self):
 
   def cmdline(self, args=()):
     cmdline = ' '.join(self._pex.cmdline(args))
-
-    def render_env_var(key, value):
-      return '{key}={value}'.format(key=key, value=value)
-
-    env_vars = [(self._PEX_PYTHON_PATH_ENV_VAR_NAME, self._pex._interpreter.binary)]
-
     pex_path = self._pex_path()
     if pex_path:
-      env_vars.append((self._PEX_PATH_ENV_VAR_NAME, pex_path))
-
-    return '{execution_control_env_vars} {cmdline}'.format(
-      execution_control_env_vars=' '.join(render_env_var(k, v) for k, v in env_vars),
-      cmdline=cmdline
-    )
+      return '{env_var_name}={pex_path} {cmdline}'.format(env_var_name=self._PEX_PATH_ENV_VAR_NAME,
+                                                          pex_path=pex_path,
+                                                          cmdline=cmdline)
+    else:
+      return cmdline
 
   def run(self, *args, **kwargs):
     env = copy(kwargs.pop('env', {}))
 
-    # Hack around bug in PEX where custom interpreters are not forwarded to PEXEnvironments.
-    # TODO(John Sirois): Remove when https://github.com/pantsbuild/pex/issues/522 is fixed.
-    env[self._PEX_PYTHON_PATH_ENV_VAR_NAME] = self._pex._interpreter.binary
-
     pex_path = self._pex_path()
     if pex_path:
       env[self._PEX_PATH_ENV_VAR_NAME] = pex_path