VCS Resolvable

pex/resolvable.py

@@ -5,6 +5,7 @@
 
 import os
 import re
+import tempfile
 from abc import abstractmethod, abstractproperty
 
 from pex.base import maybe_requirement, requirement_is_exact
@@ -13,6 +14,7 @@
 from pex.installer import Packager, SetuptoolsInstallerBase
 from pex.package import Package
 from pex.resolver_options import ResolverOptionsBuilder, ResolverOptionsInterface
+from pex.third_party.pip._internal import vcs as pip_vcs
 from pex.third_party.pkg_resources import Requirement, safe_extra
 
 # Extract extras as specified per "declaring extras":
@@ -129,32 +131,47 @@ class ResolvableRepository(Resolvable):
   """A VCS repository resolvable, e.g. 'git+', 'svn+', 'hg+', 'bzr+' packages."""
 
   COMPATIBLE_VCS = frozenset(['git', 'svn', 'hg', 'bzr'])
+  PATTERN = re.compile(r"""(.+?)\+(.+?)(?:#(.*))?""")
+
+  @staticmethod
+  def parse_query_string(query_string):
+    if query_string is None:
+      return {}
+
+    params = {
+      key: value
+      for key, value
+      in (kv_pair.split('=') for kv_pair in query_string.split('&'))}
+
+    return params
+
+  @classmethod
+  def parse_vcs_url(cls, url):
+    match = cls.PATTERN.match(url)
+
+    protocol, url, query_string = match.groups()
+
+    params = cls.parse_query_string(query_string)
+
+    return protocol, url, params
 
   @classmethod
   def from_string(cls, requirement_string, options_builder, interpreter=None):
     if any(requirement_string.startswith('%s+' % vcs) for vcs in cls.COMPATIBLE_VCS):
-      # further delegate
-      pass
+      protocol, _, params = cls.parse_vcs_url(requirement_string)
 
-    # TODO(wickman) Implement: Issue #93.
-    raise cls.InvalidRequirement('Versioning system URLs not supported.')
+      backend = pip_vcs.vcs.get_backend(protocol)
 
-  def __init__(self, options):
-    super(ResolvableRepository, self).__init__(options)
+      directory = tempfile.mkdtemp()
 
-  def compatible(self, iterator):
-    return []
+      backend.export(directory, url=requirement_string)
 
-  def packages(self):
-    return []
+      if 'subdirectory' in params:
+        directory = os.path.join(directory, params['subdirectory'])
 
-  @property
-  def name(self):
-    raise NotImplementedError
+      return ResolvableDirectory.from_string(directory, options_builder, interpreter)
 
-  @property
-  def exact(self):
-    return True
+    raise cls.InvalidRequirement('Versioning system URLs not supported.')
 
 
 class ResolvablePackage(Resolvable):

pex/vendor/__init__.py

@@ -77,6 +77,7 @@ def iter_vendor_specs():
   :return: An iterator over specs of all vendored code.
   :rtype: :class:`collection.Iterator` of :class:`VendorSpec`
   """
+  yield VendorSpec.create('pip==19.2.3')
   yield VendorSpec.create('setuptools==40.6.2')
 
   # We're currently stuck here due to removal of an API we depend on.

pex/vendor/__main__.py

@@ -5,6 +5,7 @@
 
 import os
 import pkgutil
+import re
 import subprocess
 import sys
 from collections import OrderedDict
@@ -186,9 +187,9 @@ class VendorizeError(Exception):
   """Indicates an error was encountered updating vendored libraries."""
 
 
-def vendorize(root_dir, vendor_specs, prefix):
+def vendorize(root_dir, vendor_specs, prefix, ignore_list):
   for vendor_spec in vendor_specs:
-    cmd = ['pip', 'install', '--upgrade', '--no-compile', '--target', vendor_spec.target_dir,
+    cmd = ['pip', 'install', '--no-compile', '--target', vendor_spec.target_dir,
            vendor_spec.requirement]
     result = subprocess.call(cmd)
     if result != 0:
@@ -214,6 +215,12 @@ def vendorize(root_dir, vendor_specs, prefix):
     for f in files:
       if f.endswith('.py'):
         python_file = os.path.join(root, f)
+        path = os.path.join(re.sub('^' + root_dir, '', root).strip('/'), f)
+
+        if path in ignore_list:
+          print(green('Ignoring {python_file}...'.format(python_file=python_file)))
+          continue
+
         print(green('Examining {python_file}...'.format(python_file=python_file)))
         modifications = import_rewriter.rewrite(python_file)
         if modifications:
@@ -227,14 +234,12 @@ def vendorize(root_dir, vendor_specs, prefix):
 
 
 if __name__ == '__main__':
-  if len(sys.argv) != 1:
-    print('Usage: {}'.format(sys.argv[0]), file=sys.stderr)
-    sys.exit(1)
+  ignore_list = set(sys.argv[1:])
 
   root_directory = os.path.realpath(os.path.join(os.path.dirname(__file__), '..', '..'))
   import_prefix = third_party.import_prefix()
   try:
-    vendorize(root_directory, list(iter_vendor_specs()), import_prefix)
+    vendorize(root_directory, list(iter_vendor_specs()), import_prefix, ignore_list)
     sys.exit(0)
   except VendorizeError as e:
     print('Problem encountered vendorizing: {}'.format(e), file=sys.stderr)

pex/vendor/_vendored/pip/pip-19.2.3.dist-info/INSTALLER

@@ -0,0 +1 @@
+pip

pex/vendor/_vendored/pip/pip-19.2.3.dist-info/LICENSE.txt

@@ -0,0 +1,20 @@
+Copyright (c) 2008-2019 The pip developers (see AUTHORS.txt file)
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

pex/vendor/_vendored/pip/pip-19.2.3.dist-info/METADATA

@@ -0,0 +1,81 @@
+Metadata-Version: 2.1
+Name: pip
+Version: 19.2.3
+Summary: The PyPA recommended tool for installing Python packages.
+Home-page: https://pip.pypa.io/
+Author: The pip developers
+Author-email: pypa-dev@groups.google.com
+License: MIT
+Keywords: distutils easy_install egg setuptools wheel virtualenv
+Platform: UNKNOWN
+Classifier: Development Status :: 5 - Production/Stable
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Topic :: Software Development :: Build Tools
+Classifier: Programming Language :: Python
+Classifier: Programming Language :: Python :: 2
+Classifier: Programming Language :: Python :: 2.7
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.5
+Classifier: Programming Language :: Python :: 3.6
+Classifier: Programming Language :: Python :: 3.7
+Classifier: Programming Language :: Python :: Implementation :: CPython
+Classifier: Programming Language :: Python :: Implementation :: PyPy
+Requires-Python: >=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*
+
+pip - The Python Package Installer
+==================================
+
+.. image:: https://img.shields.io/pypi/v/pip.svg
+   :target: https://pypi.org/project/pip/
+
+.. image:: https://readthedocs.org/projects/pip/badge/?version=latest
+   :target: https://pip.pypa.io/en/latest
+
+pip is the `package installer`_ for Python. You can use pip to install packages from the `Python Package Index`_ and other indexes.
+
+Please take a look at our documentation for how to install and use pip:
+
+* `Installation`_
+* `Usage`_
+
+Updates are released regularly, with a new version every 3 months. More details can be found in our documentation:
+
+* `Release notes`_
+* `Release process`_
+
+If you find bugs, need help, or want to talk to the developers please use our mailing lists or chat rooms:
+
+* `Issue tracking`_
+* `Discourse channel`_
+* `User IRC`_
+
+If you want to get involved head over to GitHub to get the source code, look at our development documentation and feel free to jump on the developer mailing lists and chat rooms:
+
+* `GitHub page`_
+* `Dev documentation`_
+* `Dev mailing list`_
+* `Dev IRC`_
+
+Code of Conduct
+---------------
+
+Everyone interacting in the pip project's codebases, issue trackers, chat
+rooms, and mailing lists is expected to follow the `PyPA Code of Conduct`_.
+
+.. _package installer: https://packaging.python.org/en/latest/current/
+.. _Python Package Index: https://pypi.org
+.. _Installation: https://pip.pypa.io/en/stable/installing.html
+.. _Usage: https://pip.pypa.io/en/stable/
+.. _Release notes: https://pip.pypa.io/en/stable/news.html
+.. _Release process: https://pip.pypa.io/en/latest/development/release-process/
+.. _GitHub page: https://github.com/pypa/pip
+.. _Dev documentation: https://pip.pypa.io/en/latest/development
+.. _Issue tracking: https://github.com/pypa/pip/issues
+.. _Discourse channel: https://discuss.python.org/c/packaging
+.. _Dev mailing list: https://groups.google.com/forum/#!forum/pypa-dev
+.. _User IRC: https://webchat.freenode.net/?channels=%23pypa
+.. _Dev IRC: https://webchat.freenode.net/?channels=%23pypa-dev
+.. _PyPA Code of Conduct: https://www.pypa.io/en/latest/code-of-conduct/
+
+

pex/vendor/_vendored/pip/pip-19.2.3.dist-info/WHEEL

@@ -0,0 +1,6 @@
+Wheel-Version: 1.0
+Generator: bdist_wheel (0.33.6)
+Root-Is-Purelib: true
+Tag: py2-none-any
+Tag: py3-none-any
+

pex/vendor/_vendored/pip/pip-19.2.3.dist-info/entry_points.txt

@@ -0,0 +1,5 @@
+[console_scripts]
+pip = pip._internal:main
+pip3 = pip._internal:main
+pip3.7 = pip._internal:main
+

pex/vendor/_vendored/pip/pip-19.2.3.dist-info/top_level.txt

@@ -0,0 +1 @@
+pip

pex/vendor/_vendored/pip/pip/__init__.py

@@ -0,0 +1 @@
+__version__ = "19.2.3"

pex/vendor/_vendored/pip/pip/__main__.py

@@ -0,0 +1,23 @@
+from __future__ import absolute_import
+
+import os
+import sys
+
+# If we are running from a wheel, add the wheel to sys.path
+# This allows the usage python pip-*.whl/pip install pip-*.whl
+if __package__ == '':
+    # __file__ is pip-*.whl/pip/__main__.py
+    # first dirname call strips of '/__main__.py', second strips off '/pip'
+    # Resulting path is the name of the wheel itself
+    # Add that to sys.path so we can import pip
+    path = os.path.dirname(os.path.dirname(__file__))
+    sys.path.insert(0, path)
+
+if "__PEX_UNVENDORED__" in __import__("os").environ:
+  from pip._internal import main as _main  # vendor:skip
+else:
+  from pex.third_party.pip._internal import main as _main
+  # isort:skip # noqa
+
+if __name__ == '__main__':
+    sys.exit(_main())

pex/vendor/_vendored/pip/pip/_internal/__init__.py

@@ -0,0 +1,109 @@
+#!/usr/bin/env python
+from __future__ import absolute_import
+
+import locale
+import logging
+import os
+import warnings
+
+import sys
+
+# 2016-06-17 barry@debian.org: urllib3 1.14 added optional support for socks,
+# but if invoked (i.e. imported), it will issue a warning to stderr if socks
+# isn't available.  requests unconditionally imports urllib3's socks contrib
+# module, triggering this warning.  The warning breaks DEP-8 tests (because of
+# the stderr output) and is just plain annoying in normal usage.  I don't want
+# to add socks as yet another dependency for pip, nor do I want to allow-stderr
+# in the DEP-8 tests, so just suppress the warning.  pdb tells me this has to
+# be done before the import of pip.vcs.
+if "__PEX_UNVENDORED__" in __import__("os").environ:
+  from pip._vendor.urllib3.exceptions import DependencyWarning  # vendor:skip
+else:
+  from pex.third_party.pip._vendor.urllib3.exceptions import DependencyWarning
+
+warnings.filterwarnings("ignore", category=DependencyWarning)  # noqa
+
+# We want to inject the use of SecureTransport as early as possible so that any
+# references or sessions or what have you are ensured to have it, however we
+# only want to do this in the case that we're running on macOS and the linked
+# OpenSSL is too old to handle TLSv1.2
+try:
+    import ssl
+except ImportError:
+    pass
+else:
+    # Checks for OpenSSL 1.0.1 on MacOS
+    if sys.platform == "darwin" and ssl.OPENSSL_VERSION_NUMBER < 0x1000100f:
+        try:
+            if "__PEX_UNVENDORED__" in __import__("os").environ:
+              from pip._vendor.urllib3.contrib import securetransport  # vendor:skip
+            else:
+              from pex.third_party.pip._vendor.urllib3.contrib import securetransport
+
+        except (ImportError, OSError):
+            pass
+        else:
+            securetransport.inject_into_urllib3()
+
+if "__PEX_UNVENDORED__" in __import__("os").environ:
+  from pip._internal.cli.autocompletion import autocomplete  # vendor:skip
+else:
+  from pex.third_party.pip._internal.cli.autocompletion import autocomplete
+
+if "__PEX_UNVENDORED__" in __import__("os").environ:
+  from pip._internal.cli.main_parser import parse_command  # vendor:skip
+else:
+  from pex.third_party.pip._internal.cli.main_parser import parse_command
+
+if "__PEX_UNVENDORED__" in __import__("os").environ:
+  from pip._internal.commands import commands_dict  # vendor:skip
+else:
+  from pex.third_party.pip._internal.commands import commands_dict
+
+if "__PEX_UNVENDORED__" in __import__("os").environ:
+  from pip._internal.exceptions import PipError  # vendor:skip
+else:
+  from pex.third_party.pip._internal.exceptions import PipError
+
+if "__PEX_UNVENDORED__" in __import__("os").environ:
+  from pip._internal.utils import deprecation  # vendor:skip
+else:
+  from pex.third_party.pip._internal.utils import deprecation
+
+if "__PEX_UNVENDORED__" in __import__("os").environ:
+  from pip._vendor.urllib3.exceptions import InsecureRequestWarning  # vendor:skip
+else:
+  from pex.third_party.pip._vendor.urllib3.exceptions import InsecureRequestWarning
+
+
+logger = logging.getLogger(__name__)
+
+# Hide the InsecureRequestWarning from urllib3
+warnings.filterwarnings("ignore", category=InsecureRequestWarning)
+
+
+def main(args=None):
+    if args is None:
+        args = sys.argv[1:]
+
+    # Configure our deprecation warnings to be sent through loggers
+    deprecation.install_warning_logger()
+
+    autocomplete()
+
+    try:
+        cmd_name, cmd_args = parse_command(args)
+    except PipError as exc:
+        sys.stderr.write("ERROR: %s" % exc)
+        sys.stderr.write(os.linesep)
+        sys.exit(1)
+
+    # Needed for locale.getpreferredencoding(False) to work
+    # in pip._internal.utils.encoding.auto_decode
+    try:
+        locale.setlocale(locale.LC_ALL, '')
+    except locale.Error as e:
+        # setlocale can apparently crash if locale are uninitialized
+        logger.debug("Ignoring error %s when setting locale", e)
+    command = commands_dict[cmd_name](isolated=("--isolated" in cmd_args))
+    return command.main(cmd_args)