Strip trailing zeros in required versions when necessary (improves co…

…mpatibility with pip)

py2deb/__init__.py

@@ -3,7 +3,7 @@
 # Authors:
 #  - Arjan Verwer
 #  - Peter Odding <peter.odding@paylogic.com>
-# Last Change: June 28, 2015
+# Last Change: September 4, 2015
 # URL: https://py2deb.readthedocs.org
 
 """
@@ -15,4 +15,4 @@
 """
 
 # Semi-standard module versioning.
-__version__ = '0.23.1'
+__version__ = '0.23.2'

py2deb/converter.py

@@ -3,7 +3,7 @@
 # Authors:
 #  - Arjan Verwer
 #  - Peter Odding <peter.odding@paylogic.com>
-# Last Change: April 21, 2015
+# Last Change: September 4, 2015
 # URL: https://py2deb.readthedocs.org
 
 """
@@ -20,6 +20,7 @@
 # Standard library modules.
 import logging
 import os
+import re
 import shutil
 import tempfile
 
@@ -28,13 +29,14 @@
 from deb_pkg_tools.cache import get_default_cache
 from deb_pkg_tools.checks import check_duplicate_files
 from deb_pkg_tools.utils import find_debian_architecture
-from humanfriendly import coerce_boolean
+from humanfriendly import coerce_boolean, compact
 from pip_accel import PipAccelerator
 from pip_accel.config import Config as PipAccelConfig
 from six.moves import configparser
 
 # Modules included in our package.
-from py2deb.utils import compact_repeating_words, normalize_package_name, PackageRepository
+from py2deb.utils import (compact_repeating_words, normalize_package_name, normalize_package_version,
+                          package_names_match, PackageRepository, tokenize_version)
 from py2deb.package import PackageToConvert
 
 # Initialize a logger.
@@ -405,8 +407,12 @@ def convert(self, pip_install_arguments):
         try:
             generated_archives = []
             dependencies_to_report = []
-            # Download, unpack and convert no-yet-converted packages.
-            for package in self.get_source_distributions(pip_install_arguments):
+            # Download and unpack the requirement set and store the complete
+            # set as an instance variable because transform_version() will need
+            # it later on.
+            self.packages_to_convert = list(self.get_source_distributions(pip_install_arguments))
+            # Convert packages that haven't been converted already.
+            for package in self.packages_to_convert:
                 # If the requirement is a 'direct' (non-transitive) requirement
                 # it means the caller explicitly asked for this package to be
                 # converted, so we add it to the list of converted dependencies
@@ -503,6 +509,91 @@ def transform_name(self, python_package_name, *extras):
         # Always normalize the package name (even if it was given to us by the caller).
         return normalize_package_name(debian_package_name)
 
+    def transform_version(self, package_to_convert, python_requirement_name, python_requirement_version):
+        """
+        Transform a Python requirement version to a Debian version number.
+
+        :param package_to_convert: The :class:`.PackageToConvert` whose
+                                   requirement is being transformed.
+        :param python_requirement_name: The name of a Python package
+                                        as found on PyPI (a string).
+        :param python_requirement_version: The required version of the
+                                           Python package (a string).
+        :returns: The transformed version (a string).
+
+        This method is a wrapper for :func:`.normalize_package_version()` that
+        takes care of one additional quirk to ensure compatibility with pip.
+        Explaining this quirk requires a bit of context:
+
+        - When package A requires package B (via ``install_requires``) and
+          package A absolutely pins the required version of package B using one
+          or more trailing zeros (e.g. ``B==1.0.0``) but the actual version
+          number of package B (embedded in the metadata of package B) contains
+          less trailing zeros (e.g. ``1.0``) then pip will not complain but
+          silently fetch version ``1.0`` of package B to satisfy the
+          requirement.
+
+        - However this doesn't change the absolutely pinned version in the
+          ``install_requires`` metadata of package A.
+
+        - When py2deb converts the resulting requirement set, the dependency of
+          package A is converted as ``B (= 1.0.0)``. The resulting packages
+          will not be installable because ``apt`` considers ``1.0`` to be
+          different from ``1.0.0``.
+
+        This method analyzes the requirement set to identify occurrences of
+        this quirk and strip trailing zeros in ``install_requires`` metadata
+        that would otherwise result in converted packages that cannot be
+        installed.
+        """
+        matching_packages = [p for p in self.packages_to_convert
+                             if package_names_match(p.python_name, python_requirement_name)]
+        if len(matching_packages) != 1:
+            # My assumption while writing this code is that this should never
+            # happen. This check is to make sure that if it does happen it will
+            # be noticed because the last thing I want is for this `hack' to
+            # result in packages that are silently wrongly converted.
+            normalized_name = normalize_package_name(python_requirement_name)
+            num_matches = len(matching_packages)
+            raise Exception(compact("""
+                Expected requirement set to contain exactly one Python package
+                whose name can be normalized to {name} but encountered {count}
+                packages instead! (matching packages: {matches})
+            """, name=normalized_name, count=num_matches, matches=matching_packages))
+        # Check whether the version number included in the requirement set
+        # matches the version number in a package's requirements.
+        requirement_to_convert = matching_packages[0]
+        if python_requirement_version != requirement_to_convert.python_version:
+            logger.debug("Checking whether to strip trailing zeros from required version ..")
+            # Check whether the version numbers share the same prefix.
+            required_version = tokenize_version(python_requirement_version)
+            included_version = tokenize_version(requirement_to_convert.python_version)
+            common_length = min(len(required_version), len(included_version))
+            required_prefix = required_version[:common_length]
+            included_prefix = included_version[:common_length]
+            prefixes_match = (required_prefix == included_prefix)
+            logger.debug("Prefix of required version: %s", required_prefix)
+            logger.debug("Prefix of included version: %s", included_prefix)
+            logger.debug("Prefixes match? %s", prefixes_match)
+            # Check if 1) only the required version has a suffix and 2) this
+            # suffix consists only of trailing zeros.
+            required_suffix = required_version[common_length:]
+            included_suffix = included_version[common_length:]
+            logger.debug("Suffix of required version: %s", required_suffix)
+            logger.debug("Suffix of included version: %s", included_suffix)
+            if prefixes_match and required_suffix and not included_suffix:
+                # Check whether the suffix of the required version contains
+                # only zeros, i.e. pip considers the version numbers the same
+                # although apt would not agree.
+                if all(re.match('^0+$', t) for t in required_suffix if t.isdigit()):
+                    modified_version = ''.join(required_prefix)
+                    logger.warning("Stripping superfluous trailing zeros from required"
+                                   " version of %s required by %s! (%s -> %s)",
+                                   python_requirement_name, package_to_convert.python_name,
+                                   python_requirement_version, modified_version)
+                    python_requirement_version = modified_version
+        return normalize_package_version(python_requirement_version)
+
     @cached_property
     def debian_architecture(self):
         """

py2deb/package.py

@@ -3,7 +3,7 @@
 # Authors:
 #  - Arjan Verwer
 #  - Peter Odding <peter.odding@paylogic.com>
-# Last Change: April 21, 2015
+# Last Change: September 4, 2015
 # URL: https://py2deb.readthedocs.org
 
 """
@@ -35,7 +35,7 @@
 from six.moves import configparser
 
 # Modules included in our package.
-from py2deb.utils import (embed_install_prefix, package_names_match, normalize_package_version,
+from py2deb.utils import (embed_install_prefix, normalize_package_version, package_names_match,
                           python_version, TemporaryDirectory)
 
 # Initialize a logger.
@@ -122,10 +122,10 @@ def debian_version(self):
         """
         The version of the Debian package (a string).
 
-        Reformats the version of the Python package using
-        :py:func:`.normalize_package_version()`.
+        Reformats :attr:`python_version` using
+        :func:`.normalize_package_version()`.
         """
-        return normalize_package_version(self.requirement.version)
+        return normalize_package_version(self.python_version)
 
     @cached_property
     def debian_maintainer(self):
@@ -292,7 +292,7 @@ def debian_dependencies(self):
             debian_package_name = self.converter.transform_name(requirement.project_name, *requirement.extras)
             if requirement.specs:
                 for constraint, version in requirement.specs:
-                    version = normalize_package_version(version)
+                    version = self.converter.transform_version(self, requirement.project_name, version)
                     if version == 'dev':
                         # Requirements like 'pytz > dev' (celery==3.1.16) don't
                         # seem to really mean anything to pip (based on my

py2deb/tests.py

@@ -1,7 +1,7 @@
 # Automated tests for the `py2deb' package.
 #
 # Author: Peter Odding <peter.odding@paylogic.com>
-# Last Change: April 21, 2015
+# Last Change: September 4, 2015
 # URL: https://py2deb.readthedocs.org
 
 """
@@ -25,7 +25,6 @@
 import shutil
 import sys
 import tempfile
-import textwrap
 import unittest
 
 # External dependencies.
@@ -34,6 +33,7 @@
 from deb_pkg_tools.control import load_control_file
 from deb_pkg_tools.package import inspect_package, parse_filename
 from executor import execute
+from humanfriendly import dedent
 
 # Modules included in our package.
 from py2deb.cli import main
@@ -371,6 +371,41 @@ def test_conversion_of_binary_package(self):
             # dpkg-shlibdeps was run successfully).
             assert 'libc6' in metadata['Depends'].names
 
+    def test_install_requires_version_munging(self):
+        """
+        Convert a package with a requirement whose version is "munged" by pip.
+
+        Refer to :func:`py2deb.converter.PackageConverter.transform_version()`
+        for details about the purpose of this test.
+        """
+        with TemporaryDirectory() as repository_directory:
+            with TemporaryDirectory() as distribution_directory:
+                # Create a temporary (and rather trivial :-) Python package.
+                with open(os.path.join(distribution_directory, 'setup.py'), 'w') as handle:
+                    handle.write(dedent('''
+                        from setuptools import setup
+                        setup(
+                            name='install-requires-munging-test',
+                            version='1.0',
+                            install_requires=['humanfriendly==1.30.0'],
+                        )
+                    '''))
+                # Run the conversion command.
+                converter = self.create_isolated_converter()
+                converter.set_repository(repository_directory)
+                archives, relationships = converter.convert([distribution_directory])
+                # Find the generated *.deb archive.
+                pathname = find_package_archive(archives, 'python-install-requires-munging-test')
+                # Use deb-pkg-tools to inspect the package metadata.
+                metadata, contents = inspect_package(pathname)
+                logger.debug("Metadata of generated package: %s", dict(metadata))
+                logger.debug("Contents of generated package: %s", dict(contents))
+                # Inspect the converted package's dependency.
+                assert metadata['Depends'].matches('python-humanfriendly', '1.30'), \
+                    "py2deb failed to rewrite version of dependency!"
+                assert not metadata['Depends'].matches('python-humanfriendly', '1.30.0'), \
+                    "py2deb failed to rewrite version of dependency!"
+
     def test_conversion_of_isolated_packages(self):
         """
         Convert a group of packages with a custom name and installation prefix.
@@ -416,7 +451,7 @@ def test_conversion_with_configuration_file(self):
         with TemporaryDirectory() as directory:
             configuration_file = os.path.join(directory, 'py2deb.ini')
             with open(configuration_file, 'w') as handle:
-                handle.write(format('''
+                handle.write(dedent('''
                     [py2deb]
                     repository = {repository}
                     name-prefix = pip-accel
@@ -643,14 +678,3 @@ def find_file(contents, pattern):
             matches.append(metadata)
     assert len(matches) == 1, "Expected to match exactly one archive entry!"
     return matches[0]
-
-
-def format(text, **kw):
-    """
-    Dedent, strip and format a multiline string with format specifications.
-
-    :param text: The text to format (a string).
-    :param kw: Any format string arguments.
-    :returns: The formatted text (a string).
-    """
-    return textwrap.dedent(text).strip().format(**kw)

py2deb/utils.py

@@ -3,7 +3,7 @@
 # Authors:
 #  - Arjan Verwer
 #  - Peter Odding <peter.odding@paylogic.com>
-# Last Change: March 4, 2015
+# Last Change: September 4, 2015
 # URL: https://py2deb.readthedocs.org
 
 """
@@ -25,6 +25,9 @@
 # Initialize a logger.
 logger = logging.getLogger(__name__)
 
+integer_pattern = re.compile('([0-9]+)')
+"""Compiled regular expression to match a consecutive run of digits."""
+
 
 class PackageRepository(object):
 
@@ -197,6 +200,16 @@ def normalize_package_version(python_package_version):
     return sanitized_version
 
 
+def tokenize_version(version_number):
+    """
+    Tokenize a string containing a version number.
+
+    :param version_number: The string to tokenize.
+    :returns: A list of strings.
+    """
+    return [t for t in integer_pattern.split(version_number) if t]
+
+
 def package_names_match(a, b):
     """
     Check whether two Python package names are equal.

requirements.txt

@@ -2,7 +2,7 @@ cached-property >= 0.1.5
 coloredlogs >= 0.5
 deb-pkg-tools >= 1.30
 executor >= 1.2
-humanfriendly >= 1.31
+humanfriendly >= 1.33
 pip-accel >= 0.25
 pkginfo >= 1.1
 six >= 1.6.1