Upgrade to pex 1.6.6.

This allows us to get rid of resolving setuptools and wheel in our
interpreter cache.

Fixes #6927

3rdparty/python/requirements.txt

@@ -2,7 +2,7 @@ ansicolors==1.0.2
 asttokens==1.1.13
 beautifulsoup4>=4.6.0,<4.7
 cffi==1.11.5
-configparser==3.5.0 ; python_version<'3'
+configparser==3.7.1 ; python_version<'3'
 contextlib2==0.5.5
 coverage>=4.5,<4.6
 docutils>=0.12,<0.13
@@ -17,7 +17,7 @@ more-itertools<6.0.0 ; python_version<'3'
 packaging==16.8
 parameterized==0.6.1
 pathspec==0.5.9
-pex==1.5.3
+pex==1.6.6
 psutil==5.4.8
 pycodestyle==2.4.0
 pyflakes==2.0.0
@@ -33,7 +33,7 @@ requests[security]>=2.20.1
 responses==0.10.4
 scandir==1.2
 setproctitle==1.1.10
-setuptools==40.4.3
+setuptools==40.6.3
 six>=1.9.0,<2
 subprocess32==3.2.7 ; python_version<'3'
 wheel==0.31.1

build-support/bin/release.sh

@@ -82,17 +82,13 @@ function run_pex() {
   # TODO: Cache this in case we run pex multiple times
   (
     PEX_VERSION="$(requirement pex | sed -e "s|pex==||")"
-    PEX_PEX=pex27
-    if [[ "${python_three:-false}" == "true" ]]; then
-      PEX_PEX=pex36
-    fi
 
     pexdir="$(mktemp -d -t build_pex.XXXXX)"
     trap "rm -rf ${pexdir}" EXIT
 
-    pex="${pexdir}/${PEX_PEX}"
+    pex="${pexdir}/pex"
 
-    curl -sSL "${PEX_DOWNLOAD_PREFIX}/v${PEX_VERSION}/${PEX_PEX}" > "${pex}"
+    curl -sSL "${PEX_DOWNLOAD_PREFIX}/v${PEX_VERSION}/pex" > "${pex}"
     chmod +x "${pex}"
     "${pex}" "$@"
   )
@@ -658,8 +654,12 @@ function build_pex() {
     platform_flags=("${platform_flags[@]}" "--platform=${platform}")
   done
 
+  # Pants depends on twitter.common libraries that trigger pex warnings for not properly declaring
+  # their dependency on setuptools (for namespace package support). To prevent these known warnings
+  # from polluting stderr we pass `--no-emit-warnings`.
   execute_pex \
     -o "${dest}" \
+    --no-emit-warnings \
     --script=pants \
     --interpreter-constraint="${interpreter_constraint}" \
     "${platform_flags[@]}" \

contrib/awslambda/python/src/python/pants/contrib/awslambda/python/subsystems/lambdex.py

@@ -9,5 +9,11 @@
 
 class Lambdex(PythonToolBase):
   options_scope = 'lambdex'
-  default_requirements = ['lambdex==0.1.2']
+  default_requirements = [
+    'lambdex==0.1.3',
+
+    # TODO(John Sirois): Remove when we can upgrade to a version of lambdex with
+    # https://github.com/wickman/lambdex/issues/6 fixed.
+    'setuptools==40.8.0'
+  ]
   default_entry_point = 'lambdex.bin.lambdex'

src/python/pants/backend/python/interpreter_cache.py

@@ -7,14 +7,11 @@
 import logging
 import os
 import shutil
-from builtins import map, str
+from builtins import str
 from collections import defaultdict
 
 from pex.interpreter import PythonInterpreter
-from pex.package import EggPackage, Package, SourcePackage
-from pex.resolver import resolve
 
-from pants.backend.python.subsystems.python_repos import PythonRepos
 from pants.backend.python.subsystems.python_setup import PythonSetup
 from pants.backend.python.targets.python_target import PythonTarget
 from pants.base.exceptions import TaskError
@@ -27,24 +24,14 @@
 logger = logging.getLogger(__name__)
 
 
-# TODO(wickman) Create a safer version of this and add to twitter.common.dirutil
-def _safe_link(src, dst):
-  try:
-    os.unlink(dst)
-  except OSError:
-    pass
-  os.symlink(src, dst)
-
-
 # TODO: Move under subsystems/ .
 class PythonInterpreterCache(Subsystem):
   """Finds python interpreters on the local system."""
   options_scope = 'python-interpreter-cache'
 
   @classmethod
   def subsystem_dependencies(cls):
-    return (super(PythonInterpreterCache, cls).subsystem_dependencies() +
-            (PythonSetup, PythonRepos))
+    return super(PythonInterpreterCache, cls).subsystem_dependencies() + (PythonSetup,)
 
   class UnsatisfiableInterpreterConstraintsError(TaskError):
     """Indicates a python interpreter matching given constraints could not be located."""
@@ -63,10 +50,6 @@ def _matching(cls, interpreters, filters=()):
   def _python_setup(self):
     return PythonSetup.global_instance()
 
-  @memoized_property
-  def _python_repos(self):
-    return PythonRepos.global_instance()
-
   @memoized_property
   def _cache_dir(self):
     cache_dir = self._python_setup.interpreter_cache_dir
@@ -124,17 +107,17 @@ def _interpreter_from_relpath(self, path, filters=()):
         return None
     except OSError:
       return None
-    interpreter = PythonInterpreter.from_binary(executable, include_site_extras=False)
+    interpreter = PythonInterpreter.from_binary(executable)
     if self._matches(interpreter, filters=filters):
-      return self._resolve(interpreter)
+      return interpreter
     return None
 
   def _setup_interpreter(self, interpreter, identity_str):
     cache_target_path = os.path.join(self._cache_dir, identity_str)
     with safe_concurrent_creation(cache_target_path) as safe_path:
       os.mkdir(safe_path)  # Parent will already have been created by safe_concurrent_creation.
       os.symlink(interpreter.binary, os.path.join(safe_path, 'python'))
-      return self._resolve(interpreter, safe_path)
+      return interpreter
 
   def _setup_cached(self, filters=()):
     """Find all currently-cached interpreters."""
@@ -193,67 +176,6 @@ def unsatisfied_filters():
       'Initialized Python interpreter cache with {}'.format(', '.join([x.binary for x in matches])))
     return matches
 
-  def _resolve(self, interpreter, interpreter_dir=None):
-    """Resolve and cache an interpreter with a setuptools and wheel capability."""
-    interpreter = self._resolve_interpreter(interpreter, interpreter_dir,
-                                            self._python_setup.setuptools_requirement())
-    if interpreter:
-      return self._resolve_interpreter(interpreter, interpreter_dir,
-                                       self._python_setup.wheel_requirement())
-
-  def _resolve_interpreter(self, interpreter, interpreter_dir, requirement):
-    """Given a :class:`PythonInterpreter` and a requirement, return an interpreter with the
-    capability of resolving that requirement or ``None`` if it's not possible to install a
-    suitable requirement.
-
-    If interpreter_dir is unspecified, operates on the default location.
-    """
-    if interpreter.satisfies([requirement]):
-      return interpreter
-
-    if not interpreter_dir:
-      interpreter_dir = os.path.join(self._cache_dir, str(interpreter.identity))
-
-    target_link = os.path.join(interpreter_dir, requirement.key)
-    bdist = self._resolve_and_link(interpreter, requirement, target_link)
-    if bdist:
-      return interpreter.with_extra(bdist.name, bdist.raw_version, bdist.path)
-    else:
-      logger.debug('Failed to resolve requirement {} for {}'.format(requirement, interpreter))
-
-  def _resolve_and_link(self, interpreter, requirement, target_link):
-    # Short-circuit if there is a local copy.
-    if os.path.exists(target_link) and os.path.exists(os.path.realpath(target_link)):
-      bdist = Package.from_href(os.path.realpath(target_link))
-      if bdist.satisfies(requirement):
-        return bdist
-
-    # Since we're resolving to bootstrap a bare interpreter, we won't have wheel available.
-    # Explicitly set the precedence to avoid resolution of wheels or distillation of sdists into
-    # wheels.
-    precedence = (EggPackage, SourcePackage)
-    resolved_dists = resolve(requirements=[requirement],
-                             fetchers=self._python_repos.get_fetchers(),
-                             interpreter=interpreter,
-                             # The local interpreter cache is, by definition, composed of
-                             #  interpreters for the 'current' platform.
-                             platform='current',
-                             context=self._python_repos.get_network_context(),
-                             precedence=precedence)
-    if not resolved_dists:
-      return None
-
-    assert len(resolved_dists) == 1, ('Expected exactly 1 distribution to be resolved for {}, '
-                                     'found:\n\t{}'.format(requirement,
-                                                           '\n\t'.join(map(str, resolved_dists))))
-
-    dist_location = resolved_dists[0].distribution.location
-    target_location = os.path.join(os.path.dirname(target_link), os.path.basename(dist_location))
-    shutil.move(dist_location, target_location)
-    _safe_link(target_location, target_link)
-    logger.debug('    installed {}'.format(target_location))
-    return Package.from_href(target_location)
-
   def _purge_interpreter(self, interpreter_dir):
     try:
       logger.info('Detected stale interpreter `{}` in the interpreter cache, purging.'

src/python/pants/backend/python/rules/python_test_runner.py

@@ -34,17 +34,9 @@ def run_python_test(transitive_hydrated_target, pytest):
   target_root = transitive_hydrated_target.root
 
   # TODO: Inject versions and digests here through some option, rather than hard-coding it.
-  interpreter_major, interpreter_minor = sys.version_info[0:2]
-  pex_name, digest = {
-    (2, 7): ("pex27", Digest('0ecbf48e3e240a413189194a9f829aec10446705c84db310affe36e23e741dbc', 1812737)),
-    (3, 6): ("pex36", Digest('ba865e7ce7a840070d58b7ba24e7a67aff058435cfa34202abdd878e7b5d351d', 1812158)),
-    (3, 7): ("pex37", Digest('51bf8e84d5290fe5ff43d45be78d58eaf88cf2a5e995101c8ff9e6a73a73343d', 1813189))
-  }.get((interpreter_major, interpreter_minor), (None, None))
-  if pex_name is None:
-    raise ValueError("Current interpreter {}.{} is not supported, as there is no corresponding PEX to download.".format(interpreter_major, interpreter_minor))
-
-  pex_snapshot = yield Get(Snapshot,
-    UrlToFetch("https://github.com/pantsbuild/pex/releases/download/v1.6.1/{}".format(pex_name), digest))
+  url = 'https://github.com/pantsbuild/pex/releases/download/v1.6.6/pex'
+  digest = Digest('61bb79384db0da8c844678440bd368bcbfac17bbdb865721ad3f9cb0ab29b629', 1826945)
+  pex_snapshot = yield Get(Snapshot, UrlToFetch(url, digest))
 
   all_targets = [target_root] + [dep.root for dep in transitive_hydrated_target.dependencies]
 

src/python/pants/backend/python/subsystems/BUILD

@@ -5,7 +5,6 @@ python_library(
   dependencies = [
     '3rdparty/python:future',
     '3rdparty/python:pex',
-    '3rdparty/python:setuptools',
     '3rdparty/python/twitter/commons:twitter.common.collections',
     'src/python/pants/base:build_environment',
     'src/python/pants/base:exceptions',