Integrate PEX interpreter selection based on target-level interpreter compatibility constraints

3rdparty/python/requirements.txt

@@ -13,7 +13,7 @@ packaging==16.8
 pathspec==0.5.0
 parameterized==0.6.1
 pep8==1.6.2
-pex==1.2.13
+pex==1.2.15
 psutil==4.3.0
 pyflakes==1.1.0
 Pygments==1.4

src/python/pants/backend/python/interpreter_cache.py

@@ -11,6 +11,7 @@
 from pex.interpreter import PythonIdentity, PythonInterpreter
 from pex.package import EggPackage, Package, SourcePackage
 from pex.resolver import resolve
+from pex.variables import Variables
 
 from pants.backend.python.targets.python_target import PythonTarget
 from pants.base.exceptions import TaskError
@@ -39,6 +40,22 @@ def _matching(cls, interpreters, filters):
       if cls._matches(interpreter, filters):
         yield interpreter
 
+  @classmethod
+  def pex_python_paths(cls):
+    """A list of paths to Python interpreter binaries as defined by a
+    PEX_PYTHON_PATH defined in either in '/etc/pexrc', '~/.pexrc'.
+    PEX_PYTHON_PATH defines a colon-seperated list of paths to interpreters
+    that a pex can be built and ran against.
+
+    :return: paths to interpreters as specified by PEX_PYTHON_PATH
+    :rtype: list
+    """
+    ppp = Variables.from_rc().get('PEX_PYTHON_PATH')
+    if ppp:
+      return ppp.split(os.pathsep)
+    else:
+      return []
+
   def __init__(self, python_setup, python_repos, logger=None):
     self._python_setup = python_setup
     self._python_repos = python_repos
@@ -132,6 +149,7 @@ def setup(self, paths=(), filters=(b'',)):
     # an escape hatch.
     filters = filters if any(filters) else self._python_setup.interpreter_constraints
     setup_paths = (paths
+                   or self.pex_python_paths()
                    or self._python_setup.interpreter_search_paths
                    or os.getenv('PATH').split(os.pathsep))
 

src/python/pants/backend/python/subsystems/python_setup.py

@@ -64,7 +64,9 @@ def register_options(cls, register):
                   'If unspecified, a standard path under the workdir is used.')
     register('--interpreter-search-paths', advanced=True, type=list, default=[],
              metavar='<binary-paths>',
-             help='A list of paths to search for python interpreters.')
+             help='A list of paths to search for python interpreters. Note that if a PEX_PYTHON_PATH '
+              'variable is defined in a pexrc file, those interpreter paths will take precedence over ' 
+              'this option.')
 
   @property
   def interpreter_constraints(self):

src/python/pants/backend/python/tasks2/python_binary_create.py

@@ -116,6 +116,10 @@ def _create_binary(self, binary_tgt, results_dir):
       for tgt in binary_tgt.closure(exclude_scopes=Scopes.COMPILE):
         if has_python_sources(tgt) or has_resources(tgt):
           source_tgts.append(tgt)
+          # Add target's interpreter compatibility constraints to pex info.
+          if has_python_sources(tgt):
+            for constraint in tgt.compatibility:
+              builder.add_interpreter_constraint(constraint)
         elif has_python_requirements(tgt):
           req_tgts.append(tgt)
 

src/python/pants/backend/python/tasks2/python_execution_task_base.py

@@ -16,6 +16,7 @@
 from pants.backend.python.targets.python_requirement_library import PythonRequirementLibrary
 from pants.backend.python.targets.python_target import PythonTarget
 from pants.backend.python.tasks2.gather_sources import GatherSources
+from pants.backend.python.tasks2.pex_build_util import has_python_sources
 from pants.backend.python.tasks2.resolve_requirements import ResolveRequirements
 from pants.backend.python.tasks2.resolve_requirements_task_base import ResolveRequirementsTaskBase
 from pants.build_graph.address import Address
@@ -141,6 +142,11 @@ def create_pex(self, pex_info=None):
 
         with safe_concurrent_creation(path) as safe_path:
           builder = PEXBuilder(safe_path, interpreter, pex_info=pex_info)
+          # Add target interpreter compatibilities to pex info.
+          for rt in relevant_targets:
+            if has_python_sources(rt):
+              for constraint in rt.compatibility:
+                builder.add_interpreter_constraint(constraint)
           builder.freeze()
 
     return WrappedPEX(PEX(os.path.realpath(path), interpreter), interpreter)

src/python/pants/backend/python/tasks2/select_interpreter.py

@@ -51,6 +51,9 @@ def execute(self):
     python_tgts = self.context.targets(lambda tgt: isinstance(tgt, PythonTarget))
     fs = PythonInterpreterFingerprintStrategy()
     with self.invalidated(python_tgts, fingerprint_strategy=fs) as invalidation_check:
+      if PythonSetup.global_instance().interpreter_search_paths and PythonInterpreterCache.pex_python_paths:
+        self.context.log.warn("Detected both PEX_PYTHON_PATH and --python-setup-interpreter-search-paths. "
+                              "Ignoring --python-setup-interpreter-search-paths.")
       # If there are no relevant targets, we still go through the motions of selecting
       # an interpreter, to prevent downstream tasks from having to check for this special case.
       if invalidation_check.all_vts:

testprojects/src/python/interpreter_selection/BUILD

@@ -9,7 +9,7 @@ python_library(
   sources = ['echo_interpreter_version.py'],
   dependencies = [],
   # Play with this to test interpreter selection in the pex machinery.
-  compatibility = ['CPython>=2.7,<3', 'CPython>=3.3']
+  compatibility = ['CPython>=2.7,<4']
 )
 
 python_binary(

testprojects/src/python/interpreter_selection/python_3_selection_testing/BUILD

@@ -0,0 +1,69 @@
+# Copyright 2017 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+# This BUILD file is intended to be an example of python 3 compatible targets and
+# to serve as a set of test cases for interpreter selection based on different targets.
+# To successfully build and run this binary target, the user will need to define a 
+# .pexrc file in /etx/pexrc, ~/.pexrc, or in a .pexrc in the pants root dir. The pexrc 
+# will need to contain a PEX_PYTHON_PATH variable containing an absolute path to a 
+# python 3 interpreter.
+#
+# An example of a PEX_PYTHON_PATH variable in a pexrc:
+# 	PEX_PYTHON_PATH=/path/to/python2.7:/path/to/python3.6
+#
+# Note that the basename of the python binaries specified must either be 'python' or 
+# 'pythonX.Y'. Specifing only major version (i.e. python3) will be ignored by Pants 
+# interpreter filtration.
+# 
+
+python_binary(
+  name='main_py3',
+  source='main_py3.py',
+  compatibility=['CPython>3'],
+  dependencies=[
+    ':lib_py3'
+  ]
+)
+
+python_library(
+  name='lib_py3',
+  sources=['lib_py3.py'],
+  compatibility=['CPython>3']
+)
+
+python_tests(
+  name='test_py3',
+  sources=[
+    'test_py3.py',
+  ],
+  dependencies=[
+    ':main_py3'
+  ],
+  compatibility=['CPython>3']
+)
+
+python_binary(
+  name='main_py2',
+  source='main_py2.py',
+  compatibility=['CPython>2.7.6,<3'],
+  dependencies=[
+    ':lib_py2'
+  ]
+)
+
+python_library(
+  name='lib_py2',
+  sources=['lib_py2.py'],
+  compatibility=['CPython>2.7.6,<3']
+)
+
+python_tests(
+  name='test_py2',
+  sources=[
+    'test_py2.py',
+  ],
+  dependencies=[
+    ':main_py2'
+  ],
+  compatibility=['CPython>2.7.6,<3']
+)

testprojects/src/python/interpreter_selection/python_3_selection_testing/lib_py2.py

@@ -0,0 +1,20 @@
+# coding=utf-8
+# Copyright 2017 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+from __future__ import (absolute_import, division, generators, nested_scopes, print_function,
+                        unicode_literals, with_statement)
+
+
+# A simple example to include in a python 2 binary target
+
+def say_something():
+  print('I am a python 2 library method.')
+  # Note that ascii exists as a built-in in Python 3 and
+  # does not exist in Python 2.
+  try:
+    ret = ascii
+  except NameError:
+    ret = None
+  assert ret is None
+  return ret

testprojects/src/python/interpreter_selection/python_3_selection_testing/lib_py3.py

@@ -0,0 +1,9 @@
+# coding=utf-8
+# Copyright 2017 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+# A simple example to include in a python 3 binary target
+
+def say_something():
+  print('I am a python 3 library method.')
+  return ascii

testprojects/src/python/interpreter_selection/python_3_selection_testing/main_py2.py

@@ -0,0 +1,23 @@
+# coding=utf-8
+# Copyright 2017 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+from __future__ import (absolute_import, division, generators, nested_scopes, print_function,
+                        unicode_literals, with_statement)
+
+import sys
+
+from interpreter_selection.python_3_selection_testing.lib_py2 import say_something
+
+
+# A simple example to test building/running/testing a python 2 binary target
+
+
+def main():
+  v = sys.version_info
+  print(sys.executable)
+  print('%d.%d.%d' % v[0:3])
+  return say_something()
+
+if __name__ == '__main__':
+  main()

testprojects/src/python/interpreter_selection/python_3_selection_testing/main_py3.py

@@ -0,0 +1,20 @@
+# coding=utf-8
+# Copyright 2017 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+import sys
+
+from interpreter_selection.python_3_selection_testing.lib_py3 import say_something
+
+
+# A simple example to test building/running/testing a python 3 binary target
+
+
+def main():
+  v = sys.version_info
+  print(sys.executable)
+  print('%d.%d.%d' % v[0:3])
+  return say_something()
+
+if __name__ == '__main__':
+  main()

testprojects/src/python/interpreter_selection/python_3_selection_testing/test_py2.py

@@ -0,0 +1,18 @@
+# coding=utf-8
+# Copyright 2017 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+from __future__ import (absolute_import, division, generators, nested_scopes, print_function,
+                        unicode_literals, with_statement)
+
+import sys
+
+from interpreter_selection.python_3_selection_testing.main_py2 import main
+
+
+def test_main():
+  print(sys.executable)
+  # Note that ascii exists as a built-in in Python 3 and
+  # does not exist in Python 2
+  ret = main()
+  assert ret == None

testprojects/src/python/interpreter_selection/python_3_selection_testing/test_py3.py

@@ -0,0 +1,15 @@
+# coding=utf-8
+# Copyright 2017 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+import sys
+
+from interpreter_selection.python_3_selection_testing.main_py3 import main
+
+
+def test_main():
+  print(sys.executable)
+  # Note that ascii exists as a built-in in Python 3 and
+  # does not exist in Python 2
+  ret = main()
+  assert ret == ascii

tests/python/pants_test/BUILD

@@ -56,6 +56,7 @@ python_library(
     'src/python/pants/util:dirutil',
     'src/python/pants/util:process_handler',
     'tests/python/pants_test/testutils:file_test_util',
+    'tests/python/pants_test/testutils:pexrc_util',
   ]
 )
 

tests/python/pants_test/backend/python/BUILD

@@ -59,5 +59,9 @@ python_tests(
     'src/python/pants/python',
     'src/python/pants/util:contextutil',
     'tests/python/pants_test:base_test',
+    'tests/python/pants_test:int-test',
+    'tests/python/pants_test/testutils:git_util',
+    'tests/python/pants_test/testutils:pexrc_util',
   ],
+  timeout=1200
 )

tests/python/pants_test/backend/python/tasks2/BUILD

@@ -40,7 +40,8 @@ python_tests(
     'src/python/pants/util:contextutil',
     'src/python/pants/util:process_handler',
     'tests/python/pants_test:int-test',
+    '3rdparty/python:pex',
   ],
   tags={'integration'},
-  timeout=600
+  timeout=2400
 )

tests/python/pants_test/backend/python/tasks2/test_pytest_run_integration.py

@@ -6,13 +6,17 @@
                         unicode_literals, with_statement)
 
 import os
+import sys
 import time
 
 from pants.util.contextutil import temporary_dir
 from pants_test.pants_run_integration_test import PantsRunIntegrationTest
+from pants_test.testutils.pexrc_util import setup_pexrc_with_pex_python_path
 
 
 class PytestRunIntegrationTest(PantsRunIntegrationTest):
+  testproject = 'testprojects/src/python/interpreter_selection'
+
   def test_pytest_run_timeout_succeeds(self):
     pants_run = self.run_pants(['clean-all',
                                 'test.pytest',
@@ -95,3 +99,30 @@ def test_pytest_with_profile(self):
       # We won't see a profile at prof itself because PANTS_PROFILE wasn't set when the
       # current process started.
       self.assertTrue(os.path.exists('{}.0'.format(prof)))
+
+  def test_pants_test_interpreter_selection_with_pexrc(self):
+    """Test the pants test goal with intepreters selected from a PEX_PYTHON_PATH
+    defined in a pexrc file on disk.
+
+    """
+    py27 = '2.7'
+    py3 = '3'
+    if self.has_python_version(py27) and self.has_python_version(py3):
+      print('Found both python {} and python {}. Running test.'.format(py27, py3))
+      py27_path, py3_path = self.python_interpreter_path(py27), self.python_interpreter_path(py3)
+      with setup_pexrc_with_pex_python_path(os.path.join(os.path.dirname(sys.argv[0]), '.pexrc'), [py27_path, py3_path]):
+        with temporary_dir() as interpreters_cache:
+          pants_ini_config = {'python-setup': {'interpreter_cache_dir': interpreters_cache}}
+          pants_run_27 = self.run_pants(
+            command=['test', '{}:test_py2'.format(os.path.join(self.testproject, 'python_3_selection_testing'))],
+            config=pants_ini_config
+          )
+          self.assert_success(pants_run_27)
+          pants_run_3 = self.run_pants(
+            command=['test', '{}:test_py3'.format(os.path.join(self.testproject, 'python_3_selection_testing'))],
+            config=pants_ini_config
+          )
+          self.assert_success(pants_run_3)
+    else:
+      print('Could not find both python {} and python {} on system. Skipping.'.format(py27, py3))
+      self.skipTest('Missing neccesary Python interpreters on system.')