ordering of requirements matters

.travis.yml

@@ -15,14 +15,7 @@ matrix:
         # pylint has dropped 2.6 support -.-
         - python: "2.6"
           env: TOXENV=lint
-install:
-    - pip install -r requirements.d/travis.txt
-    # My tests need some package to be reliably installed to system-site-packages.
-    # This should maybe be a fixture of some kind,
-    # but I'd have to build python from source I think =/
-    - SYS_REAL_PREFIX=$(python -c 'import sys; print(sys.real_prefix)')
-    - echo $SYS_REAL_PREFIX
-    - sudo $SYS_REAL_PREFIX/bin/pip install virtualenv
+install: pip install -r requirements.d/travis.txt
 script: tox
 after_success:
     - find -name '.coverage*' | xargs mv -t .
@@ -32,11 +25,12 @@ after_failure:
     # attempt to show any pip.log on failure
     - find -name pip.log | xargs -r tail -n99999
 
-sudo: true
+
+# public caches only supported under sudo:false currently
+#   closest thing to documentation:
+#   http://blog.travis-ci.com/2014-10-07-free-builds-for-students-github-student-developer-pack
+sudo: false
 cache:
-    # public caches only supported under sudo:false currently
-    #   closest thing to documentation:
-    #   http://blog.travis-ci.com/2014-10-07-free-builds-for-students-github-student-developer-pack
     directories:
         - $HOME/.pip
         - $HOME/.pre-commit

TODO

@@ -5,8 +5,7 @@ Should probably do before it hits users
 
 Things that I want to do, but would put me past my deadline:
 
-* coverage: 155, 209, 226, 230-232, 303-304, 384, 388
-* coverage: 155, 208, 225, 229-231, 338, 368, 372
+* coverage: 115, 155, 193, 258, 275, 286-289, 427, 431
 
 * populate wheels into the cache during build
     see also: https://github.com/pypa/pip/pull/1572

requirements.d/_lint.txt

@@ -1,12 +1,7 @@
 # packages needed for linting, but not for prod
 flake8
-pre-commit
-
 # pylint  # PR merged: https://bitbucket.org/logilab/pylint/pull-request/186/fixed_up_import_test-and-tests/diff
 # pending fresh release: https://bitbucket.org/logilab/pylint/issue/363/please-cut-a-release-131
 # note: pylint is set up such that -e simply doesn't work. Their __init__ is in their top directory.
 hg+https://bitbucket.org/logilab/pylint@58c66aa083777059a2e6b46f6a0545a2f4977097#egg=pylint
-
-# astroid 1.3.0 is broken.
-# see: https://bitbucket.org/logilab/astroid/issue/55/astroid-130-explodes-on-import-pytest
-astroid!=1.3.0
+pre-commit

requirements.txt

@@ -1,2 +1 @@
 -r requirements.d/prod.txt
-pep8==1.0

tests/functional/get_installed_test.py

@@ -9,7 +9,7 @@ def get_installed(capfd):
 
     venv_update_script('''\
 import venv_update as v
-for p in sorted(v.pip_get_installed()):
+for p in sorted(v.reqnames(v.pip_get_installed())):
     print(p)''', venv='myvenv')
 
     out, err = capfd.readouterr()

tests/functional/install_test.py

@@ -14,8 +14,8 @@ def test_pip_install_flake8(tmpdir, capfd):
 
     venv_update_script('''\
 import json
-from venv_update import pip_install
-print(json.dumps(sorted(pip_install(('flake8',)))))
+from venv_update import pip_install, reqnames
+print(json.dumps(sorted(reqnames(pip_install(('flake8',))))))
 ''', venv='myvenv')
 
     out, err = capfd.readouterr()

tests/functional/relocation_test.py

@@ -34,42 +34,3 @@ def test_is_relocatable_different_python_version(tmpdir):
     venv_update(python_arg)
 
     run('sh', '-c', '. virtualenv_run/bin/activate && doge --help')
-
-
-def _get_virtualenv_data(capfd):
-    out, err = capfd.readouterr()  # flush buffers
-    Path('assertions.py').write('''
-import json
-import sys, virtualenv
-print(json.dumps((virtualenv.__file__, sys.prefix, sys.real_prefix)))
-''')
-    run('virtualenv_run/bin/python', 'assertions.py')
-
-    out, err = capfd.readouterr()
-    assert err == ''
-
-    from json import loads
-    lastline = out.splitlines()[-1]
-    return loads(lastline)
-
-
-def path_is_within(path, within):
-    from os.path import relpath
-    return not relpath(path, within).startswith('..')
-
-
-def test_is_relocatable_system_site_packages(tmpdir, capfd):
-    tmpdir.chdir()
-    requirements = Path('requirements.txt')
-
-    # first show that virtualenv is installed to the system python
-    # then show that virtualenv is installed to the virtualenv python, when it's required
-    for reqs, invenv in (
-            ('', False),
-            ('virtualenv\n', True),
-    ):
-        requirements.write(reqs)
-        venv_update('--system-site-packages')
-        virtualenv__file__, prefix, real_prefix = _get_virtualenv_data(capfd)
-        assert path_is_within(virtualenv__file__, prefix) == invenv
-        assert path_is_within(virtualenv__file__, real_prefix) == (not invenv)

tests/functional/simple_test.py

@@ -4,7 +4,7 @@
 import pytest
 
 from testing import (
-    get_scenario, run, strip_coverage_warnings, venv_update, venv_update_script, TOP,
+    get_scenario, run, strip_coverage_warnings, venv_update, TOP,
 )
 
 from sys import version_info
@@ -27,6 +27,7 @@ def test_second_install_faster(tmpdir):
         # An arbitrary package that takes a bit of time to install: twisted
         # Should I make my own fake c-extention just to remove this dependency?
         requirements.write('''\
+pudb==2014.1
 simplejson
 pyyaml==3.11
 pylint
@@ -241,29 +242,3 @@ def test_uncolored_pipe():
     out = pipe_output(read, write)
 
     assert not unprintable(out), out
-
-
-@pytest.mark.parametrize("options,expected", [
-    (('--system-site-packages',), True),
-    ((), False),
-])
-def test_venv_uses_system_site_packages(capfd, tmpdir, options, expected):
-    tmpdir.chdir()
-    Path('requirements.txt').write('')
-
-    for options, expected in (
-            (options, expected),
-            # also show that going back and forth works
-            ((), False),
-            (('--system-site-packages',), True),
-    ):
-        venv_update(*options)
-
-        out, err = capfd.readouterr()  # flush buffers
-        venv_update_script('''\
-from venv_update import venv_uses_system_site_packages
-print(venv_uses_system_site_packages())''')
-
-        out, err = capfd.readouterr()  # flush buffers
-        assert err == ''
-        assert out == '%s\n' % expected

tests/unit/simple_test.py

@@ -36,13 +36,40 @@ def test_parse_reqs(tmpdir):
         reqs.write('''\
 pep8''')
 
-    assert set(
-        venv_update.pip_parse_requirements(('reqs.txt',))
-    ) == set(
-        ['mccabe', 'pep8', 'aweirdname', 'cov-core', None, 'pep8']
-    )
+    # show that ordering is preserved in the parse
+    parsed = venv_update.pip_parse_requirements(('reqs.txt',))
+    assert [
+        (req.name, req.url)
+        for req in parsed
+    ] == [
+        (None, 'file://' + tmpdir.strpath),
+        ('pep8', None),
+        ('mccabe', None),
+        ('pep8', None),
+        ('aweirdname', 'hg+https://bitbucket.org/bukzor/coverage.py@__main__-support#egg=aweirdname'),
+        ('cov-core', 'git+git://github.com/bukzor/cov-core.git@master#egg=cov-core'),
+        (None, 'hg+https://bitbucket.org/logilab/pylint@58c66aa083777059a2e6b46f6a0545a2f4977097'),
+        (None, 'file:///my/random/project'),
+        (None, 'file:///my/random/project2'),
+    ]
+
+    # cheat: also unit-test format_req:
+    assert [
+        venv_update.format_req(req)
+        for req in parsed
+    ] == [
+        ('file://' + tmpdir.strpath,),
+        ('pep8',),
+        ('mccabe',),
+        ('pep8==1.0',),
+        ('-e', 'hg+https://bitbucket.org/bukzor/coverage.py@__main__-support#egg=aweirdname'),
+        ('-e', 'git+git://github.com/bukzor/cov-core.git@master#egg=cov-core'),
+        ('hg+https://bitbucket.org/logilab/pylint@58c66aa083777059a2e6b46f6a0545a2f4977097',),
+        ('file:///my/random/project',),
+        ('-e', 'file:///my/random/project2'),
+    ]
 
 
 def test_pip_get_installed():
     installed = venv_update.pip_get_installed()
-    assert 'venv-update' in installed
+    assert 'venv-update' in venv_update.reqnames(installed)

venv_update.py

@@ -155,40 +155,34 @@ def pip(args):
         exit(result)
 
 
+def dist_to_req(dist):
+    """Make a pip.FrozenRequirement from a pkg_resources distribution object"""
+    from pip import FrozenRequirement
+    # TODO: does it matter that we completely ignore dependency_links?
+    return FrozenRequirement.from_dist(dist, [])
+
+
 def pip_get_installed():
     """Code extracted from the middle of the pip freeze command.
     """
-    from pip import FrozenRequirement
-    from pip._vendor.pkg_resources import working_set
     from pip.util import get_installed_distributions
 
-    dependency_links = []
-
-    for dist in working_set:
-        if dist.has_metadata('dependency_links.txt'):
-            dependency_links.extend(
-                dist.get_metadata_lines('dependency_links.txt')
-            )
-
-    installed = {}
+    installed = []
     for dist in get_installed_distributions(local_only=True):
-        req = FrozenRequirement.from_dist(
-            dist,
-            dependency_links,
-        )
-
-        installed[req.name] = req
+        req = dist_to_req(dist)
+        installed.append(req)
 
     return installed
 
 
 def pip_parse_requirements(requirement_files):
     from pip.req import parse_requirements
 
-    required = {}
+    # ordering matters =/
+    required = []
     for reqfile in requirement_files:
         for req in parse_requirements(reqfile):
-            required[req.name] = req
+            required.append(req)
     return required
 
 
@@ -212,12 +206,12 @@ def exactly_satisfied(pipreq):
 
 
 def filter_exactly_satisfied(reqs):
-    result = {}
-    for pkg, req in reqs.items():
+    result = []
+    for req in reqs:
         if exactly_satisfied(req):
             print('Requirement already up-to-date: %s' % req)
         else:
-            result[pkg] = req
+            result.append(req)
     return result
 
 
@@ -261,9 +255,9 @@ def install(self, options, args):
         InstallCommand.run = orig_installcommand['run']
 
     if _nonlocal.successfully_installed is None:
-        return {}
+        return []
     else:
-        return dict(_nonlocal.successfully_installed.requirements)
+        return _nonlocal.successfully_installed.requirements.values()
 
 
 def trace_requirements(requirements):
@@ -273,7 +267,7 @@ def trace_requirements(requirements):
     from pip._vendor.pkg_resources import get_provider, DistributionNotFound
 
     stack = list(requirements)
-    result = {}
+    result = []
     while stack:
         req = stack.pop()
         if req is None:
@@ -283,15 +277,14 @@ def trace_requirements(requirements):
         try:
             dist = get_provider(req)
         except (DistributionNotFound, IOError):
-            result[req.project_name] = None
             continue
 
-        result[dist.project_name] = dist
+        result.append(dist_to_req(dist))
 
         try:
             dist_reqs = dist.requires()  # should we support extras?
         except IOError:
-            # This happens sometimes with setuptools. Don't understand why, yet.
+            # This happens sometimes with setuptools. I don't understand why, yet.
             #   IOError: [Errno 2] No such file or directory: '${site-packages}/setuptools-3.6.dist-info/METADATA'
             continue
 
@@ -302,6 +295,10 @@ def trace_requirements(requirements):
     return result
 
 
+def reqnames(reqs):
+    return set(req.name for req in reqs)
+
+
 @contextmanager
 def venv(venv_path, venv_args):
     """Ensure we have a virtualenv."""
@@ -317,12 +314,6 @@ def venv(venv_path, venv_args):
     )
 
 
-def venv_uses_system_site_packages():
-    from glob import glob
-    from sys import prefix
-    return not glob(prefix + '/lib/python*/no-global-site-packages.txt')
-
-
 def do_install(reqs):
     from os import environ
 
@@ -333,7 +324,7 @@ def do_install(reqs):
     requirements_as_options = sum(
         (
             format_req(req)
-            for pkg, req in unsatisfied.items()
+            for req in unsatisfied
         ),
         ()
     )
@@ -355,24 +346,18 @@ def do_install(reqs):
     # 3) Install: Use our well-populated cache to do the installations.
     # --use-wheel is somewhat redundant here, but it means we get an error if we have a bad version of pip/setuptools.
     install_opts = ('--upgrade', '--use-wheel',) + cache_opts
-    if venv_uses_system_site_packages():
-        # In the worst case, a --system-site-packages venv will be missing packages because it was built in an
-        # environment which satisfied requirements at the system level, then be relocated to a system that doesn't have
-        # the requirement, breaking it.  This has bitten us before. --ignore-installed avoids the issue.
-        print('Detected --system-site-packages; using --ignore-installed. This makes things slower.')
-        install_opts += ('--ignore-installed',)
 
     recently_installed = pip_install(install_opts + requirements_as_options)
 
     required_with_deps = trace_requirements(
-        (req.req for req in required.values())
+        (req.req for req in required)
     )
 
     # TODO-TEST require A==1 then A==2
     extraneous = (
-        set(previously_installed) -
-        set(required_with_deps) -
-        set(recently_installed)
+        reqnames(previously_installed) -
+        reqnames(required_with_deps) -
+        reqnames(recently_installed)
     )
 
     # 4) Uninstall any extraneous packages.