Update piptools to avoid reusing InstallRequirement

- Prevents re-preparation of deleted `PKG-INFO` files
- Fixes #2435

Signed-off-by: Dan Ryan <dan@danryan.co>

news/2480.bugfix

@@ -0,0 +1 @@
+Resolved a long-standing issue with re-using previously generated ``InstallRequirement`` objects for resolution which could cause ``PKG-INFO`` file information to be deleted, raising a ``TypeError``.

pipenv/patched/piptools/resolver.py

@@ -15,7 +15,7 @@
 from .cache import DependencyCache
 from .exceptions import UnsupportedConstraint
 from .logging import log
-from .utils import (format_requirement, format_specifier, full_groupby, dedup,
+from .utils import (format_requirement, format_specifier, full_groupby, dedup, simplify_markers,
                     is_pinned_requirement, key_from_ireq, key_from_req, UNSAFE_PACKAGES)
 
 green = partial(click.style, fg='green')
@@ -274,18 +274,18 @@ def _iter_dependencies(self, ireq):
         Editable requirements will never be looked up, as they may have
         changed at any time.
         """
+        _iter_ireq = simplify_markers(ireq)
         if ireq.editable:
-            for dependency in self.repository.get_dependencies(ireq):
+            for dependency in self.repository.get_dependencies(_iter_ireq):
                 yield dependency
             return
         elif ireq.markers:
-            for dependency in self.repository.get_dependencies(ireq):
+            for dependency in self.repository.get_dependencies(_iter_ireq):
                 dependency.prepared = False
                 yield dependency
-            return
         elif ireq.extras:
             valid_markers = default_environment().keys()
-            for dependency in self.repository.get_dependencies(ireq):
+            for dependency in self.repository.get_dependencies(_iter_ireq):
                 dependency.prepared = False
                 if dependency.markers and not any(dependency.markers._markers[0][0].value.startswith(k) for k in valid_markers):
                     dependency.markers = None
@@ -296,7 +296,6 @@ def _iter_dependencies(self, ireq):
                         ireq.extras = ireq.extra
 
                 yield dependency
-            return
         elif not is_pinned_requirement(ireq):
             raise TypeError('Expected pinned or editable requirement, got {}'.format(ireq))
 

pipenv/patched/piptools/utils.py

@@ -2,6 +2,7 @@
 from __future__ import (absolute_import, division, print_function,
                         unicode_literals)
 
+import six
 import os
 import sys
 from itertools import chain, groupby
@@ -13,12 +14,52 @@
 from first import first
 from pipenv.patched.notpip._vendor.packaging.specifiers import SpecifierSet, InvalidSpecifier
 from pipenv.patched.notpip._vendor.packaging.version import Version, InvalidVersion, parse as parse_version
+from pipenv.patched.notpip._vendor.packaging.markers import Marker, Op, Value, Variable
 from .click import style
 
 
 UNSAFE_PACKAGES = {'setuptools', 'distribute', 'pip'}
 
 
+def simplify_markers(ireq):
+    """simplify_markers "This code cleans up markers for a specific :class:`~InstallRequirement`"
+
+    Clean and deduplicate markers.
+
+    :param ireq: An InstallRequirement to clean
+    :type ireq: :class:`~pip._internal.req.req_install.InstallRequirement`
+    :return: An InstallRequirement with cleaned Markers
+    :rtype: :class:`~pip._internal.req.req_install.InstallRequirement`
+    """
+
+    if not getattr(ireq, 'markers', None):
+        return ireq
+    markers = ireq.markers
+    marker_list = []
+    if isinstance(markers, six.string_types):
+        if ';' in markers:
+            markers = [Marker(m_str.strip()) for m_str in markers.split(';')]
+        else:
+            markers = Marker(markers)
+    for m in markers._markers:
+        _single_marker = []
+        if isinstance(m[0], six.string_types):
+            continue
+        if not isinstance(m[0], (list, tuple)):
+            marker_list.append(''.join([_piece.serialize() for _piece in m]))
+            continue
+        for _marker_part in m:
+            if isinstance(_marker_part, six.string_types):
+                _single_marker.append(_marker_part)
+                continue
+            _single_marker.append(''.join([_piece.serialize() for _piece in _marker_part]))
+        _single_marker = [_m.strip() for _m in _single_marker]
+        marker_list.append(tuple(_single_marker,))
+    marker_str = ' and '.join(list(dedup(tuple(marker_list,)))) if marker_list else ''
+    new_markers = Marker(marker_str)
+    return make_install_requirement(ireq.name, first(ireq.specifier).version, ireq.extras, new_markers, constraint=ireq.constraint)
+
+
 def clean_requires_python(candidates):
     """Get a cleaned list of all the candidates with valid specifiers in the `requires_python` attributes."""
     all_candidates = []

pipenv/resolver.py

@@ -34,7 +34,6 @@ def main():
             new_sys_argv.append(v)
     sys.argv = new_sys_argv
 
-    from .utils import create_mirror_source, resolve_deps, replace_pypi_sources
     os.environ['PIP_PYTHON_VERSION'] = '.'.join([str(s) for s in sys.version_info[:3]])
     os.environ['PIP_PYTHON_PATH'] = sys.executable
     if is_verbose:
@@ -49,6 +48,7 @@ def main():
         for i, package in enumerate(packages):
             if package.startswith('--'):
                 del packages[i]
+    from pipenv.utils import create_mirror_source, resolve_deps, replace_pypi_sources
     pypi_mirror_source = create_mirror_source(os.environ['PIPENV_PYPI_MIRROR']) if 'PIPENV_PYPI_MIRROR' in os.environ else None
 
     def resolve(packages, pre, project, sources, verbose, clear, system):

tasks/vendoring/patches/patched/piptools.patch

@@ -19,7 +19,7 @@ index 4e6174c..75f9b49 100644
  # NOTE
  # We used to store the cache dir under ~/.pip-tools, which is not the
 diff --git a/pipenv/patched/piptools/repositories/pypi.py b/pipenv/patched/piptools/repositories/pypi.py
-index 1c4b943..c922be1 100644
+index 1c4b943..c922be1 10064
 --- a/pipenv/patched/piptools/repositories/pypi.py
 +++ b/pipenv/patched/piptools/repositories/pypi.py
 @@ -4,6 +4,7 @@ from __future__ import (absolute_import, division, print_function,
@@ -406,7 +406,7 @@ index 1c4b943..c922be1 100644
      def allow_all_wheels(self):
          """
 diff --git a/pipenv/patched/piptools/resolver.py b/pipenv/patched/piptools/resolver.py
-index 05ec8fd..c5eb728 100644
+index 05ec8fd..465414c 100644
 --- a/pipenv/patched/piptools/resolver.py
 +++ b/pipenv/patched/piptools/resolver.py
 @@ -8,13 +8,14 @@ from itertools import chain, count
@@ -421,7 +421,7 @@ index 05ec8fd..c5eb728 100644
  from .exceptions import UnsupportedConstraint
  from .logging import log
 -from .utils import (format_requirement, format_specifier, full_groupby,
-+from .utils import (format_requirement, format_specifier, full_groupby, dedup,
++from .utils import (format_requirement, format_specifier, full_groupby, dedup, simplify_markers,
                      is_pinned_requirement, key_from_ireq, key_from_req, UNSAFE_PACKAGES)
 
  green = partial(click.style, fg='green')
@@ -473,20 +473,23 @@ index 05ec8fd..c5eb728 100644
                  # Return a sorted, de-duped tuple of extras
                  combined_ireq.extras = tuple(sorted(set(tuple(combined_ireq.extras) + tuple(ireq.extras))))
              yield combined_ireq
-@@ -271,6 +276,25 @@ class Resolver(object):
+@@ -269,10 +274,28 @@ class Resolver(object):
+         Editable requirements will never be looked up, as they may have
+         changed at any time.
          """
++        _iter_ireq = simplify_markers(ireq)
          if ireq.editable:
-             for dependency in self.repository.get_dependencies(ireq):
-+                yield dependency
-+            return
+-            for dependency in self.repository.get_dependencies(ireq):
++            for dependency in self.repository.get_dependencies(_iter_ireq):
+                 yield dependency
+             return
 +        elif ireq.markers:
-+            for dependency in self.repository.get_dependencies(ireq):
++            for dependency in self.repository.get_dependencies(_iter_ireq):
 +                dependency.prepared = False
 +                yield dependency
-+            return
 +        elif ireq.extras:
 +            valid_markers = default_environment().keys()
-+            for dependency in self.repository.get_dependencies(ireq):
++            for dependency in self.repository.get_dependencies(_iter_ireq):
 +                dependency.prepared = False
 +                if dependency.markers and not any(dependency.markers._markers[0][0].value.startswith(k) for k in valid_markers):
 +                    dependency.markers = None
@@ -496,10 +499,11 @@ index 05ec8fd..c5eb728 100644
 +                    else:
 +                        ireq.extras = ireq.extra
 +
-                 yield dependency
-             return
++                yield dependency
          elif not is_pinned_requirement(ireq):
-@@ -283,14 +307,25 @@ class Resolver(object):
+             raise TypeError('Expected pinned or editable requirement, got {}'.format(ireq))
+
+@@ -283,14 +306,25 @@ class Resolver(object):
          if ireq not in self.dependency_cache:
              log.debug('  {} not in cache, need to check index'.format(format_requirement(ireq)), fg='yellow')
              dependencies = self.repository.get_dependencies(ireq)
@@ -541,22 +545,70 @@ index 08dabe1..480ad1e 100644
          else:
              return self.repository.find_best_match(ireq, prereleases)
 diff --git a/pipenv/patched/piptools/utils.py b/pipenv/patched/piptools/utils.py
-index fde5816..fb71882 100644
+index fde5816..8732673 100644
 --- a/pipenv/patched/piptools/utils.py
 +++ b/pipenv/patched/piptools/utils.py
-@@ -11,13 +11,35 @@ from contextlib import contextmanager
+@@ -2,6 +2,7 @@
+ from __future__ import (absolute_import, division, print_function,
+                         unicode_literals)
+
++import six
+ import os
+ import sys
+ from itertools import chain, groupby
+@@ -11,13 +12,75 @@ from contextlib import contextmanager
  from ._compat import InstallRequirement
 
  from first import first
 -
 +from pip._vendor.packaging.specifiers import SpecifierSet, InvalidSpecifier
 +from pip._vendor.packaging.version import Version, InvalidVersion, parse as parse_version
++from pip._vendor.packaging.markers import Marker, Op, Value, Variable
  from .click import style
 
 
  UNSAFE_PACKAGES = {'setuptools', 'distribute', 'pip'}
 
 
++def simplify_markers(ireq):
++    """simplify_markers "This code cleans up markers for a specific :class:`~InstallRequirement`"
++
++    Clean and deduplicate markers.
++
++    :param ireq: An InstallRequirement to clean
++    :type ireq: :class:`~pip._internal.req.req_install.InstallRequirement`
++    :return: An InstallRequirement with cleaned Markers
++    :rtype: :class:`~pip._internal.req.req_install.InstallRequirement`
++    """
++
++    if not getattr(ireq, 'markers', None):
++        return ireq
++    markers = ireq.markers
++    marker_list = []
++    if isinstance(markers, six.string_types):
++        if ';' in markers:
++            markers = [Marker(m_str.strip()) for m_str in markers.split(';')]
++        else:
++            markers = Marker(markers)
++    for m in markers._markers:
++        _single_marker = []
++        if isinstance(m[0], six.string_types):
++            continue
++        if not isinstance(m[0], (list, tuple)):
++            marker_list.append(''.join([_piece.serialize() for _piece in m]))
++            continue
++        for _marker_part in m:
++            if isinstance(_marker_part, six.string_types):
++                _single_marker.append(_marker_part)
++                continue
++            _single_marker.append(''.join([_piece.serialize() for _piece in _marker_part]))
++        _single_marker = [_m.strip() for _m in _single_marker]
++        marker_list.append(tuple(_single_marker,))
++    marker_str = ' and '.join(list(dedup(tuple(marker_list,)))) if marker_list else ''
++    new_markers = Marker(marker_str)
++    return make_install_requirement(ireq.name, first(ireq.specifier).version, ireq.extras, new_markers, constraint=ireq.constraint)
++
++
 +def clean_requires_python(candidates):
 +    """Get a cleaned list of all the candidates with valid specifiers in the `requires_python` attributes."""
 +    all_candidates = []
@@ -581,7 +633,7 @@ index fde5816..fb71882 100644
  def key_from_ireq(ireq):
      """Get a standardized key for an InstallRequirement."""
      if ireq.req is None and ireq.link is not None:
-@@ -43,16 +65,51 @@ def comment(text):
+@@ -43,16 +106,51 @@ def comment(text):
      return style(text, fg='green')
 
 
@@ -637,7 +689,7 @@ index fde5816..fb71882 100644
 
 
  def format_requirement(ireq, marker=None):
-@@ -63,10 +120,10 @@ def format_requirement(ireq, marker=None):
+@@ -63,10 +161,10 @@ def format_requirement(ireq, marker=None):
      if ireq.editable:
          line = '-e {}'.format(ireq.link)
      else: