Refactor wheel-finding machinery

nsist/__init__.py

@@ -23,7 +23,7 @@
 from .commands import prepare_bin_directory
 from .copymodules import copy_modules
 from .nsiswriter import NSISFileWriter
-from .pypi import fetch_pypi_wheels
+from .wheels import WheelGetter
 from .util import download, text_types, get_cache_dir, normalize_path
 
 __version__ = '2.1'
@@ -351,10 +351,11 @@ def prepare_packages(self):
             os.mkdir(build_pkg_dir)
 
         # 2. Wheels specified in pypi_wheel_reqs or in paths of local_wheels
-        fetch_pypi_wheels(self.pypi_wheel_reqs, self.local_wheels, build_pkg_dir,
-                          py_version=self.py_version, bitness=self.py_bitness,
-                          extra_sources=self.extra_wheel_sources,
-                          exclude=self.exclude)
+        wg = WheelGetter(self.pypi_wheel_reqs, self.local_wheels, build_pkg_dir,
+                         py_version=self.py_version, bitness=self.py_bitness,
+                         extra_sources=self.extra_wheel_sources,
+                         exclude=self.exclude)
+        wg.get_all()
 
         # 3. Copy importable modules
         copy_modules(self.packages, build_pkg_dir,

nsist/tests/test_local_wheels.py

@@ -1,12 +1,13 @@
 import glob
 import os
+from pathlib import Path
 import platform
 import subprocess
 
 import pytest
 from testpath import assert_isfile, assert_isdir
 
-from nsist.pypi import fetch_pypi_wheels
+from nsist.wheels import WheelGetter
 
 # To exclude tests requiring network on an unplugged machine, use: pytest -m "not network"
 
@@ -17,7 +18,8 @@ def test_matching_one_pattern(tmpdir):
 
     subprocess.call(['pip', 'wheel', 'requests==2.19.1', '-w', str(td1)])
 
-    fetch_pypi_wheels([], [os.path.join(td1, '*.whl')], td2, platform.python_version(), 64)
+    wg = WheelGetter([], [os.path.join(td1, '*.whl')], td2, platform.python_version(), 64)
+    wg.get_globs()
 
     assert_isdir(os.path.join(td2, 'requests'))
     assert_isfile(os.path.join(td2, 'requests-2.19.1.dist-info', 'METADATA'))
@@ -32,44 +34,54 @@ def test_duplicate_wheel_files_raise(tmpdir):
 
     subprocess.call(['pip', 'wheel', 'requests==2.19.1', '-w', str(td1)])
 
-    with pytest.raises(ValueError, match='wheel distribution requests already included'):
-        fetch_pypi_wheels(['requests==2.19.1'],
-                          [os.path.join(td1, '*.whl')], td2, platform.python_version(), 64)
+    wg = WheelGetter(['requests==2.19.1'], [os.path.join(td1, '*.whl')], td2,
+                     platform.python_version(), 64)
+
+    with pytest.raises(ValueError, match='Multiple wheels specified'):
+        wg.get_all()
 
 def test_invalid_wheel_file_raise(tmpdir):
     td1 = str(tmpdir.mkdir('wheels'))
     td2 = str(tmpdir.mkdir('pkgs'))
 
     open(os.path.join(td1, 'notawheel.txt'), 'w+')
 
-    with pytest.raises(ValueError, match='Invalid wheel file name: notawheel.txt'):
-        fetch_pypi_wheels([], [os.path.join(td1, '*')], td2, platform.python_version(), 64)
+    wg = WheelGetter([], [os.path.join(td1, '*')], td2,
+                     platform.python_version(), 64)
+
+    with pytest.raises(ValueError, match='notawheel.txt'):
+        wg.get_globs()
 
-def test_incompatible_plateform_wheel_file_raise(tmpdir):
+def test_incompatible_platform_wheel_file_raise(tmpdir):
     td1 = str(tmpdir.mkdir('wheels'))
     td2 = str(tmpdir.mkdir('pkgs'))
 
-    open(os.path.join(td1, 'incompatiblewheel-1.0.0-py2.py3-none-linux_x86_64.whl'), 'w+')
+    Path(td1, 'incompatiblewheel-1.0.0-py2.py3-none-linux_x86_64.whl').touch()
+
+    wg = WheelGetter([], [os.path.join(td1, '*.whl')], td2,
+                     platform.python_version(), 64)
 
-    with pytest.raises(ValueError, match='{0} is not compatible with Python {1} for Windows'
-                       .format('incompatiblewheel-1.0.0-py2.py3-none-linux_x86_64.whl',
-                               platform.python_version())):
-        fetch_pypi_wheels([], [os.path.join(td1, '*.whl')], td2, platform.python_version(), 64)
+    with pytest.raises(ValueError, match='not compatible with .* win_amd64'):
+        wg.get_globs()
 
 def test_incompatible_python_wheel_file_raise(tmpdir):
     td1 = str(tmpdir.mkdir('wheels'))
     td2 = str(tmpdir.mkdir('pkgs'))
 
-    open(os.path.join(td1, 'incompatiblewheel-1.0.0-py26-none-any.whl'), 'w+')
+    Path(td1, 'incompatiblewheel-1.0.0-py26-none-any.whl').touch()
 
-    with pytest.raises(ValueError, match='{0} is not compatible with Python {1} for Windows'
-                       .format('incompatiblewheel-1.0.0-py26-none-any.whl',
-                               platform.python_version())):
-        fetch_pypi_wheels([], [os.path.join(td1, '*.whl')], td2, platform.python_version(), 64)
+    wg = WheelGetter([], [os.path.join(td1, '*.whl')], td2,
+                     platform.python_version(), 64)
+
+    with pytest.raises(ValueError, match='not compatible with Python {}'
+                       .format(platform.python_version())):
+        wg.get_globs()
 
 def test_useless_wheel_glob_path_raise(tmpdir):
     td1 = str(tmpdir.mkdir('wheels'))
     td2 = str(tmpdir.mkdir('pkgs'))
 
-    with pytest.raises(ValueError, match='does not match any wheel file'):
-        fetch_pypi_wheels([], [os.path.join(td1, '*.whl')], td2, platform.python_version(), 64)
+    wg = WheelGetter([], [os.path.join(td1, '*.whl')], td2, '3.6', 64)
+
+    with pytest.raises(ValueError, match='does not match any files'):
+        wg.get_globs()

nsist/tests/test_pypi.py

@@ -3,16 +3,17 @@
 import pytest
 from testpath import assert_isfile
 
-from nsist.pypi import (
+from nsist.wheels import (
     WheelLocator, extract_wheel, CachedRelease, merge_dir_to, NoWheelError,
+    CompatibilityScorer,
 )
 
 # To exclude tests requiring network on an unplugged machine, use: pytest -m "not network"
 
 @pytest.mark.network
 def test_download(tmpdir):
     tmpdir = str(tmpdir)
-    wd = WheelLocator("astsearch==0.1.2", "3.5.1", 64)
+    wd = WheelLocator("astsearch==0.1.2", CompatibilityScorer("3.5.1", "win_amd64"))
     wheel = wd.fetch()
     assert_isfile(wheel)
 
@@ -41,16 +42,16 @@ def test_extra_sources(tmpdir):
     expected = (src1 / 'astsearch-0.1.2-py3-none-any.whl')
     expected.touch()
     (src2 / 'astsearch-0.1.2-py3-none-win_amd64.whl').touch()
-    wl = WheelLocator("astsearch==0.1.2", "3.5.1", 64,
-                      extra_sources=[src1, src2])
+    scorer = CompatibilityScorer("3.5.1", "win_amd64")
+    wl = WheelLocator("astsearch==0.1.2", scorer, extra_sources=[src1, src2])
     assert wl.check_extra_sources() == expected
 
-    wl = WheelLocator("astsearch==0.2.0", "3.5.1", 64,
-                      extra_sources=[src1, src2])
+    wl = WheelLocator("astsearch==0.2.0", scorer, extra_sources=[src1, src2])
     assert wl.check_extra_sources() is None
 
 def test_pick_best_wheel():
-    wd = WheelLocator("astsearch==0.1.2", "3.5.1", 64)
+    wd = WheelLocator("astsearch==0.1.2",
+                      CompatibilityScorer("3.5.1", "win_amd64"))
 
     # Some of the wheel filenames below are impossible combinations - they are
     # there to test the scoring and ranking machinery.

nsist/pypi.py → nsist/wheels.py

@@ -19,20 +19,19 @@
 
 class NoWheelError(Exception): pass
 
-class WheelLocator(object):
-    def __init__(self, requirement, py_version, bitness, extra_sources=None):
-        self.requirement = requirement
-        self.py_version = py_version
-        self.bitness = bitness
-        self.extra_sources = extra_sources or []
+class CompatibilityScorer:
+    """Score wheels for a given target platform
 
-        if requirement.count('==') != 1:
-            raise ValueError("Requirement {!r} did not match name==version".format(requirement))
-        self.name, self.version = requirement.split('==', 1)
+    0 for any score means incompatible.
+    Higher numbers are more platform specific.
+    """
+    def __init__(self, py_version, platform):
+        self.py_version = py_version
+        self.platform = platform
 
     def score_platform(self, platform):
-        target = 'win_amd64' if self.bitness == 64 else 'win32'
-        d = {target: 2, 'any': 1}
+        # target = 'win_amd64' if self.bitness == 64 else 'win32'
+        d = {self.platform: 2, 'any': 1}
         return max(d.get(p, 0) for p in platform.split('.'))
 
     def score_abi(self, abi):
@@ -52,22 +51,36 @@ def score_interpreter(self, interpreter):
             }
         return max(d.get(i, 0) for i in interpreter.split('.'))
 
+    def score(self, whl_filename):
+        m = re.search(r'-([^-]+)-([^-]+)-([^-]+)\.whl$', whl_filename)
+        if not m:
+            raise ValueError("Failed to find wheel tag in %r" % whl_filename)
+
+        interpreter, abi, platform = m.group(1, 2, 3)
+        return (
+            self.score_platform(platform),
+            self.score_abi(abi),
+            self.score_interpreter(interpreter)
+        )
+
+class WheelLocator(object):
+    def __init__(self, requirement, scorer, extra_sources=None):
+        self.requirement = requirement
+        self.scorer = scorer
+        self.extra_sources = extra_sources or []
+
+        if requirement.count('==') != 1:
+            raise ValueError("Requirement {!r} did not match name==version".format(requirement))
+        self.name, self.version = requirement.split('==', 1)
+
     def pick_best_wheel(self, release_list):
         best_score = (0, 0, 0)
         best = None
         for release in release_list:
             if release.package_type != 'wheel':
                 continue
 
-            m = re.search(r'-([^-]+)-([^-]+)-([^-]+)\.whl', release.filename)
-            if not m:
-                continue
-
-            interpreter, abi, platform = m.group(1, 2, 3)
-            score = (self.score_platform(platform),
-                     self.score_abi(abi),
-                     self.score_interpreter(interpreter)
-                    )
+            score = self.scorer.score(release.filename)
             if any(s==0 for s in score):
                 # Incompatible
                 continue
@@ -251,59 +264,64 @@ def extract_wheel(whl_file, target_dir, exclude=None):
     # Clean up temporary directory
     shutil.rmtree(str(td))
 
-
-def fetch_pypi_wheels(wheels_requirements, wheels_paths, target_dir, py_version,
-                      bitness, extra_sources=None, exclude=None):
-    """
-    Gather wheels included explicitly by wheels_pypi parameter 
-    or matching glob paths given in local_wheels parameter.
-    """
-    distributions = []
-    # We try to get the wheels from wheels_pypi requirements parameter
-    for req in wheels_requirements:
-        wl = WheelLocator(req, py_version, bitness, extra_sources)
-        whl_file = wl.fetch() 
-        extract_wheel(whl_file, target_dir, exclude=exclude)
-        distributions.append(wl.name)
-    # Then from the local_wheels paths parameter
-    for glob_path in wheels_paths:
-        paths = glob.glob(glob_path)
-        if not paths:
-            raise ValueError('Error, glob path {0} does not match any wheel file'.format(glob_path))
-        for path in paths:
-            logger.info('Collecting wheel file: %s (from: %s)', os.path.basename(path), glob_path)
-            validate_wheel(path, distributions, py_version, bitness)
-            extract_wheel(path, target_dir, exclude=exclude)
-
-
-def extract_distribution_and_version(wheel_name):
-    """Extract distribution and version from a wheel file name"""
-    search = re.search(r'^([^-]+)-([^-]+)-.*\.whl$', wheel_name)
-    if not search:
-        raise ValueError('Invalid wheel file name: {0}'.format(wheel_name))
-
-    return (search.group(1), search.group(2))
-
-
-def validate_wheel(whl_path, distributions, py_version, bitness):
-    """
-    Verify that the given wheel can safely be included in the current installer.
-    If so, the given wheel info will be included in the given wheel info array.
-    If not, an exception will be raised.
-    """
-    wheel_name = os.path.basename(whl_path)
-    (distribution, version) = extract_distribution_and_version(wheel_name)
-
-    # Check that a distribution of same name has not been included before
-    if distribution in distributions:
-        raise ValueError('Error, wheel distribution {0} already included'.format(distribution))
-
-    # Check that the wheel is compatible with the installer environment
-    locator = WheelLocator('{0}=={1}'.format(distribution, version), py_version, bitness, [Path(os.path.dirname(whl_path))])
-    if not locator.check_extra_sources():
-        raise ValueError('Error, wheel {0} is not compatible with Python {1} for Windows'.format(wheel_name, py_version))
-
-    distributions.append(distribution)
+class WheelGetter:
+    def __init__(self, requirements, wheel_globs, target_dir,
+                 py_version, bitness, extra_sources=None, exclude=None):
+        self.requirements = requirements
+        self.wheel_globs = wheel_globs
+        self.target_dir = target_dir
+        target_platform = 'win_amd64' if bitness == 64 else 'win32'
+        self.scorer = CompatibilityScorer(py_version, target_platform)
+        self.extra_sources = extra_sources
+        self.exclude = exclude
+
+        self.got_distributions = {}
+
+    def get_all(self):
+        self.get_requirements()
+        self.get_globs()
+
+    def get_requirements(self):
+        for req in self.requirements:
+            wl = WheelLocator(req, self.scorer, self.extra_sources)
+            whl_file = wl.fetch()
+            extract_wheel(whl_file, self.target_dir, exclude=self.exclude)
+            self.got_distributions[wl.name] = whl_file
+
+    def get_globs(self):
+        for glob_path in self.wheel_globs:
+            paths = glob.glob(glob_path)
+            if not paths:
+                raise ValueError('Glob path {} does not match any files'
+                                 .format(glob_path))
+            for path in paths:
+                logger.info('Collecting wheel file: %s (from: %s)',
+                            os.path.basename(path), glob_path)
+                self.validate_wheel(path)
+                extract_wheel(path, self.target_dir, exclude=self.exclude)
+
+    def validate_wheel(self, whl_path):
+        """
+        Verify that the given wheel can safely be included in the current installer.
+        If so, the given wheel info will be included in the given wheel info array.
+        If not, an exception will be raised.
+        """
+        wheel_name = os.path.basename(whl_path)
+        distribution = wheel_name.split('-', 1)[0]
+
+        # Check that a distribution of same name has not been included before
+        if distribution in self.got_distributions:
+            prev_path = self.got_distributions[distribution]
+            raise ValueError('Multiple wheels specified for {}:\n  {}\n  {}'.format(
+                             distribution, prev_path, whl_path))
+
+        # Check that the wheel is compatible with the installer environment
+        scores = self.scorer.score(wheel_name)
+        if any(s == 0 for s in scores):
+            raise ValueError('Wheel {} is not compatible with Python {}, {}'
+                .format(wheel_name, self.scorer.py_version, self.scorer.platform))
+
+        self.got_distributions[distribution] = whl_path
 
 
 def is_excluded(path, exclude):