Integrate PEX interpreter selection based on target-level interpreter compatibility constraints

3rdparty/python/requirements.txt

@@ -13,7 +13,7 @@ packaging==16.8
 pathspec==0.5.0
 parameterized==0.6.1
 pep8==1.6.2
-pex==1.2.13
+pex==1.2.15
 psutil==4.3.0
 pyflakes==1.1.0
 Pygments==1.4

src/python/pants/backend/python/interpreter_cache.py

@@ -11,6 +11,7 @@
 from pex.interpreter import PythonIdentity, PythonInterpreter
 from pex.package import EggPackage, Package, SourcePackage
 from pex.resolver import resolve
+from pex.variables import Variables
 
 from pants.backend.python.targets.python_target import PythonTarget
 from pants.base.exceptions import TaskError
@@ -39,6 +40,22 @@ def _matching(cls, interpreters, filters):
       if cls._matches(interpreter, filters):
         yield interpreter
 
+  @memoized_property
+  def pex_python_paths(self):
+    """A list of paths to Python interpreter binaries as defined by a
+    PEX_PYTHON_PATH defined in either in '/etc/pexrc', '~/.pexrc', or ./.pexrc'.
+    PEX_PYTHON_PATH defines a colon-seperated list of paths to interpreters
+    that a pex can be built and ran against.
+
+    :return: paths to interpreters as specified by PEX_PYTHON_PATH
+    :rtype: list
+    """
+    ppp = Variables.from_rc().get('PEX_PYTHON_PATH', '')
+    if ppp:
+      return ppp.split(os.pathsep)
+    else:
+      return []
+
   def __init__(self, python_setup, python_repos, logger=None):
     self._python_setup = python_setup
     self._python_repos = python_repos
@@ -127,11 +144,14 @@ def setup(self, paths=(), filters=(b'',)):
     :returns: A list of cached interpreters
     :rtype: list of :class:`pex.interpreter.PythonInterpreter`
     """
+    pex_python_paths = self.pex_python_paths
     # We filter the interpreter cache itself (and not just the interpreters we pull from it)
     # because setting up some python versions (e.g., 3<=python<3.3) crashes, and this gives us
     # an escape hatch.
-    filters = filters if any(filters) else self._python_setup.interpreter_constraints
+    if not any(filters):
+      filters = self._python_setup.interpreter_constraints
     setup_paths = (paths
+                   or pex_python_paths
                    or self._python_setup.interpreter_search_paths
                    or os.getenv('PATH').split(os.pathsep))
 

src/python/pants/backend/python/subsystems/python_setup.py

@@ -64,7 +64,9 @@ def register_options(cls, register):
                   'If unspecified, a standard path under the workdir is used.')
     register('--interpreter-search-paths', advanced=True, type=list, default=[],
              metavar='<binary-paths>',
-             help='A list of paths to search for python interpreters.')
+             help='A list of paths to search for python interpreters. Note that if a PEX_PYTHON_PATH '
+              'variable is defined in a pexrc file, those interpreter paths will take precedence over ' 
+              'this option.')
 
   @property
   def interpreter_constraints(self):

src/python/pants/backend/python/tasks2/python_binary_create.py

@@ -103,6 +103,10 @@ def _create_binary(self, binary_tgt, results_dir):
 
       builder = PEXBuilder(path=tmpdir, interpreter=interpreter, pex_info=pex_info, copy=True)
 
+      # Add binary target's interpreter compatibility to pex info
+      for constraint in binary_tgt.compatibility:
+        builder.add_interpreter_constraint(constraint)
+
       if binary_tgt.shebang:
         self.context.log.info('Found Python binary target {} with customized shebang, using it: {}'
                                 .format(binary_tgt.name, binary_tgt.shebang))
@@ -116,6 +120,10 @@ def _create_binary(self, binary_tgt, results_dir):
       for tgt in binary_tgt.closure(exclude_scopes=Scopes.COMPILE):
         if has_python_sources(tgt) or has_resources(tgt):
           source_tgts.append(tgt)
+          # Add target's interpreter compatibility constraints to pex info
+          if has_python_sources(tgt):
+            for constraint in tgt.compatibility:
+              builder.add_interpreter_constraint(constraint)
         elif has_python_requirements(tgt):
           req_tgts.append(tgt)
 

src/python/pants/backend/python/tasks2/python_execution_task_base.py

@@ -16,6 +16,7 @@
 from pants.backend.python.targets.python_requirement_library import PythonRequirementLibrary
 from pants.backend.python.targets.python_target import PythonTarget
 from pants.backend.python.tasks2.gather_sources import GatherSources
+from pants.backend.python.tasks2.pex_build_util import has_python_sources
 from pants.backend.python.tasks2.resolve_requirements import ResolveRequirements
 from pants.backend.python.tasks2.resolve_requirements_task_base import ResolveRequirementsTaskBase
 from pants.build_graph.address import Address
@@ -141,6 +142,11 @@ def create_pex(self, pex_info=None):
 
         with safe_concurrent_creation(path) as safe_path:
           builder = PEXBuilder(safe_path, interpreter, pex_info=pex_info)
+          # Add target interpreter compatibilities to pex info
+          for rt in relevant_targets:
+            if has_python_sources(rt):
+              for constraint in rt.compatibility:
+                builder.add_interpreter_constraint(constraint)
           builder.freeze()
 
     return WrappedPEX(PEX(os.path.realpath(path), interpreter), interpreter)

testprojects/src/python/interpreter_selection/python_3_binary/BUILD

@@ -0,0 +1,42 @@
+# Copyright 2017 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+# This BUILD file is intended to be an example of python 3 compatible targets.
+# To successfully build and run this binary target, the user will need to define a 
+# .pexrc file in /etx/pexrc, ~/.pexrc, or in a .pexrc in the pants root dir. The pexrc 
+# will need to contain a PEX_PYTHON_PATH variable containing an absolute path to a 
+# python 3 interpreter.
+#
+# An example of a PEX_PYTHON_PATH variable in a pexrc:
+# 	PEX_PYTHON_PATH=/path/to/python2.7:/path/to/python3.6
+#
+# Note that the basename of the python binaries specified must either be 'python' or 
+# 'pythonX.Y'. Specifing only major version (i.e. python3) will be ignored by Pants 
+# interpreter filtration.
+# 
+
+python_binary(
+  name='main',
+  source='main.py',
+  compatibility=['CPython>3'],
+  dependencies=[
+    ':lib'
+  ]
+)
+
+python_library(
+  name='lib',
+  sources=['lib.py'],
+  compatibility=['CPython>3']
+)
+
+python_tests(
+  name='test',
+  sources=[
+    'test.py',
+  ],
+  dependencies=[
+    ':main'
+  ],
+  compatibility=['CPython>3']
+)

testprojects/src/python/interpreter_selection/python_3_binary/lib.py

@@ -0,0 +1,19 @@
+# coding=utf-8
+# Copyright 2017 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+from __future__ import (absolute_import, division, generators, nested_scopes, print_function,
+                        unicode_literals, with_statement)
+
+import sys
+
+
+# A simple example to include in a python 3 binary target
+
+def say_something():
+  print('I am a python 3 library method.')
+  assert 1/2 == 0.5
+
+  # Return 1/2 for testing the `./pants run` task
+  # Note that 1/2 = 0 in python 2 and 1/2 = 0.5 in python 3
+  return 1/2  

testprojects/src/python/interpreter_selection/python_3_binary/main.py

@@ -0,0 +1,23 @@
+# coding=utf-8
+# Copyright 2017 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+from __future__ import (absolute_import, division, generators, nested_scopes, print_function,
+                        unicode_literals, with_statement)
+
+import sys
+
+from interpreter_selection.python_3_binary.lib import say_something
+
+
+# A simple example to test building/running/testing a python 3 binary target
+
+
+def main():
+  v = sys.version_info
+  print(sys.executable)
+  print('%d.%d.%d' % v[0:3])
+  return say_something()
+
+if __name__ == '__main__':
+	main()

testprojects/src/python/interpreter_selection/python_3_binary/test.py

@@ -0,0 +1,15 @@
+# coding=utf-8
+# Copyright 2017 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+from __future__ import (absolute_import, division, generators, nested_scopes, print_function,
+                        unicode_literals, with_statement)
+
+import sys
+
+from interpreter_selection.python_3_binary.main import main
+
+def test_main():
+  print(sys.executable)
+  # Note that 1/2 = 0 in python 2 and 1/2 = 0.5 in python 3
+  assert main() == 0.5