diff --git a/.travis.yml b/.travis.yml index b35db32..a6e4bd5 100644 --- a/.travis.yml +++ b/.travis.yml @@ -15,14 +15,7 @@ matrix: # pylint has dropped 2.6 support -.- - python: "2.6" env: TOXENV=lint -install: - - pip install -r requirements.d/travis.txt - # My tests need some package to be reliably installed to system-site-packages. - # This should maybe be a fixture of some kind, - # but I'd have to build python from source I think =/ - - SYS_REAL_PREFIX=$(python -c 'import sys; print(sys.real_prefix)') - - echo $SYS_REAL_PREFIX - - sudo $SYS_REAL_PREFIX/bin/pip install virtualenv +install: pip install -r requirements.d/travis.txt script: tox after_success: - find -name '.coverage*' | xargs mv -t . @@ -32,11 +25,12 @@ after_failure: # attempt to show any pip.log on failure - find -name pip.log | xargs -r tail -n99999 -sudo: true + +# public caches only supported under sudo:false currently +# closest thing to documentation: +# http://blog.travis-ci.com/2014-10-07-free-builds-for-students-github-student-developer-pack +sudo: false cache: - # public caches only supported under sudo:false currently - # closest thing to documentation: - # http://blog.travis-ci.com/2014-10-07-free-builds-for-students-github-student-developer-pack directories: - $HOME/.pip - $HOME/.pre-commit diff --git a/TODO b/TODO index 78d727d..72a335b 100644 --- a/TODO +++ b/TODO @@ -5,8 +5,7 @@ Should probably do before it hits users Things that I want to do, but would put me past my deadline: -* coverage: 155, 209, 226, 230-232, 303-304, 384, 388 -* coverage: 155, 208, 225, 229-231, 338, 368, 372 +* coverage: 115, 155, 193, 258, 275, 286-289, 427, 431 * populate wheels into the cache during build see also: https://github.com/pypa/pip/pull/1572 diff --git a/requirements.d/_lint.txt b/requirements.d/_lint.txt index 8065510..bb2ccee 100644 --- a/requirements.d/_lint.txt +++ b/requirements.d/_lint.txt @@ -1,12 +1,7 @@ # packages needed for linting, but not for prod flake8 -pre-commit - # pylint # PR merged: https://bitbucket.org/logilab/pylint/pull-request/186/fixed_up_import_test-and-tests/diff # pending fresh release: https://bitbucket.org/logilab/pylint/issue/363/please-cut-a-release-131 # note: pylint is set up such that -e simply doesn't work. Their __init__ is in their top directory. hg+https://bitbucket.org/logilab/pylint@58c66aa083777059a2e6b46f6a0545a2f4977097#egg=pylint - -# astroid 1.3.0 is broken. -# see: https://bitbucket.org/logilab/astroid/issue/55/astroid-130-explodes-on-import-pytest -astroid!=1.3.0 +pre-commit diff --git a/requirements.txt b/requirements.txt index 0c4820f..90296f4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1 @@ -r requirements.d/prod.txt -pep8==1.0 diff --git a/tests/functional/get_installed_test.py b/tests/functional/get_installed_test.py index ae82945..ffddf54 100644 --- a/tests/functional/get_installed_test.py +++ b/tests/functional/get_installed_test.py @@ -9,7 +9,7 @@ def get_installed(capfd): venv_update_script('''\ import venv_update as v -for p in sorted(v.pip_get_installed()): +for p in sorted(v.reqnames(v.pip_get_installed())): print(p)''', venv='myvenv') out, err = capfd.readouterr() diff --git a/tests/functional/install_test.py b/tests/functional/install_test.py index 975e658..241c80b 100644 --- a/tests/functional/install_test.py +++ b/tests/functional/install_test.py @@ -14,8 +14,8 @@ def test_pip_install_flake8(tmpdir, capfd): venv_update_script('''\ import json -from venv_update import pip_install -print(json.dumps(sorted(pip_install(('flake8',))))) +from venv_update import pip_install, reqnames +print(json.dumps(sorted(reqnames(pip_install(('flake8',)))))) ''', venv='myvenv') out, err = capfd.readouterr() diff --git a/tests/functional/relocation_test.py b/tests/functional/relocation_test.py index 0a17631..ebe9190 100644 --- a/tests/functional/relocation_test.py +++ b/tests/functional/relocation_test.py @@ -34,42 +34,3 @@ def test_is_relocatable_different_python_version(tmpdir): venv_update(python_arg) run('sh', '-c', '. virtualenv_run/bin/activate && doge --help') - - -def _get_virtualenv_data(capfd): - out, err = capfd.readouterr() # flush buffers - Path('assertions.py').write(''' -import json -import sys, virtualenv -print(json.dumps((virtualenv.__file__, sys.prefix, sys.real_prefix))) -''') - run('virtualenv_run/bin/python', 'assertions.py') - - out, err = capfd.readouterr() - assert err == '' - - from json import loads - lastline = out.splitlines()[-1] - return loads(lastline) - - -def path_is_within(path, within): - from os.path import relpath - return not relpath(path, within).startswith('..') - - -def test_is_relocatable_system_site_packages(tmpdir, capfd): - tmpdir.chdir() - requirements = Path('requirements.txt') - - # first show that virtualenv is installed to the system python - # then show that virtualenv is installed to the virtualenv python, when it's required - for reqs, invenv in ( - ('', False), - ('virtualenv\n', True), - ): - requirements.write(reqs) - venv_update('--system-site-packages') - virtualenv__file__, prefix, real_prefix = _get_virtualenv_data(capfd) - assert path_is_within(virtualenv__file__, prefix) == invenv - assert path_is_within(virtualenv__file__, real_prefix) == (not invenv) diff --git a/tests/functional/simple_test.py b/tests/functional/simple_test.py index b81dfd2..e45fdbe 100644 --- a/tests/functional/simple_test.py +++ b/tests/functional/simple_test.py @@ -4,7 +4,7 @@ import pytest from testing import ( - get_scenario, run, strip_coverage_warnings, venv_update, venv_update_script, TOP, + get_scenario, run, strip_coverage_warnings, venv_update, TOP, ) from sys import version_info @@ -27,6 +27,7 @@ def test_second_install_faster(tmpdir): # An arbitrary package that takes a bit of time to install: twisted # Should I make my own fake c-extention just to remove this dependency? requirements.write('''\ +pudb==2014.1 simplejson pyyaml==3.11 pylint @@ -241,29 +242,3 @@ def test_uncolored_pipe(): out = pipe_output(read, write) assert not unprintable(out), out - - -@pytest.mark.parametrize("options,expected", [ - (('--system-site-packages',), True), - ((), False), -]) -def test_venv_uses_system_site_packages(capfd, tmpdir, options, expected): - tmpdir.chdir() - Path('requirements.txt').write('') - - for options, expected in ( - (options, expected), - # also show that going back and forth works - ((), False), - (('--system-site-packages',), True), - ): - venv_update(*options) - - out, err = capfd.readouterr() # flush buffers - venv_update_script('''\ -from venv_update import venv_uses_system_site_packages -print(venv_uses_system_site_packages())''') - - out, err = capfd.readouterr() # flush buffers - assert err == '' - assert out == '%s\n' % expected diff --git a/tests/unit/simple_test.py b/tests/unit/simple_test.py index fb36dd9..0c17e75 100644 --- a/tests/unit/simple_test.py +++ b/tests/unit/simple_test.py @@ -36,13 +36,40 @@ def test_parse_reqs(tmpdir): reqs.write('''\ pep8''') - assert set( - venv_update.pip_parse_requirements(('reqs.txt',)) - ) == set( - ['mccabe', 'pep8', 'aweirdname', 'cov-core', None, 'pep8'] - ) + # show that ordering is preserved in the parse + parsed = venv_update.pip_parse_requirements(('reqs.txt',)) + assert [ + (req.name, req.url) + for req in parsed + ] == [ + (None, 'file://' + tmpdir.strpath), + ('pep8', None), + ('mccabe', None), + ('pep8', None), + ('aweirdname', 'hg+https://bitbucket.org/bukzor/coverage.py@__main__-support#egg=aweirdname'), + ('cov-core', 'git+git://github.com/bukzor/cov-core.git@master#egg=cov-core'), + (None, 'hg+https://bitbucket.org/logilab/pylint@58c66aa083777059a2e6b46f6a0545a2f4977097'), + (None, 'file:///my/random/project'), + (None, 'file:///my/random/project2'), + ] + + # cheat: also unit-test format_req: + assert [ + venv_update.format_req(req) + for req in parsed + ] == [ + ('file://' + tmpdir.strpath,), + ('pep8',), + ('mccabe',), + ('pep8==1.0',), + ('-e', 'hg+https://bitbucket.org/bukzor/coverage.py@__main__-support#egg=aweirdname'), + ('-e', 'git+git://github.com/bukzor/cov-core.git@master#egg=cov-core'), + ('hg+https://bitbucket.org/logilab/pylint@58c66aa083777059a2e6b46f6a0545a2f4977097',), + ('file:///my/random/project',), + ('-e', 'file:///my/random/project2'), + ] def test_pip_get_installed(): installed = venv_update.pip_get_installed() - assert 'venv-update' in installed + assert 'venv-update' in venv_update.reqnames(installed) diff --git a/venv_update.py b/venv_update.py index 48f4601..1abe797 100644 --- a/venv_update.py +++ b/venv_update.py @@ -155,29 +155,22 @@ def pip(args): exit(result) +def dist_to_req(dist): + """Make a pip.FrozenRequirement from a pkg_resources distribution object""" + from pip import FrozenRequirement + # TODO: does it matter that we completely ignore dependency_links? + return FrozenRequirement.from_dist(dist, []) + + def pip_get_installed(): """Code extracted from the middle of the pip freeze command. """ - from pip import FrozenRequirement - from pip._vendor.pkg_resources import working_set from pip.util import get_installed_distributions - dependency_links = [] - - for dist in working_set: - if dist.has_metadata('dependency_links.txt'): - dependency_links.extend( - dist.get_metadata_lines('dependency_links.txt') - ) - - installed = {} + installed = [] for dist in get_installed_distributions(local_only=True): - req = FrozenRequirement.from_dist( - dist, - dependency_links, - ) - - installed[req.name] = req + req = dist_to_req(dist) + installed.append(req) return installed @@ -185,10 +178,11 @@ def pip_get_installed(): def pip_parse_requirements(requirement_files): from pip.req import parse_requirements - required = {} + # ordering matters =/ + required = [] for reqfile in requirement_files: for req in parse_requirements(reqfile): - required[req.name] = req + required.append(req) return required @@ -212,12 +206,12 @@ def exactly_satisfied(pipreq): def filter_exactly_satisfied(reqs): - result = {} - for pkg, req in reqs.items(): + result = [] + for req in reqs: if exactly_satisfied(req): print('Requirement already up-to-date: %s' % req) else: - result[pkg] = req + result.append(req) return result @@ -261,9 +255,9 @@ def install(self, options, args): InstallCommand.run = orig_installcommand['run'] if _nonlocal.successfully_installed is None: - return {} + return [] else: - return dict(_nonlocal.successfully_installed.requirements) + return _nonlocal.successfully_installed.requirements.values() def trace_requirements(requirements): @@ -273,7 +267,7 @@ def trace_requirements(requirements): from pip._vendor.pkg_resources import get_provider, DistributionNotFound stack = list(requirements) - result = {} + result = [] while stack: req = stack.pop() if req is None: @@ -283,15 +277,14 @@ def trace_requirements(requirements): try: dist = get_provider(req) except (DistributionNotFound, IOError): - result[req.project_name] = None continue - result[dist.project_name] = dist + result.append(dist_to_req(dist)) try: dist_reqs = dist.requires() # should we support extras? except IOError: - # This happens sometimes with setuptools. Don't understand why, yet. + # This happens sometimes with setuptools. I don't understand why, yet. # IOError: [Errno 2] No such file or directory: '${site-packages}/setuptools-3.6.dist-info/METADATA' continue @@ -302,6 +295,10 @@ def trace_requirements(requirements): return result +def reqnames(reqs): + return set(req.name for req in reqs) + + @contextmanager def venv(venv_path, venv_args): """Ensure we have a virtualenv.""" @@ -317,12 +314,6 @@ def venv(venv_path, venv_args): ) -def venv_uses_system_site_packages(): - from glob import glob - from sys import prefix - return not glob(prefix + '/lib/python*/no-global-site-packages.txt') - - def do_install(reqs): from os import environ @@ -333,7 +324,7 @@ def do_install(reqs): requirements_as_options = sum( ( format_req(req) - for pkg, req in unsatisfied.items() + for req in unsatisfied ), () ) @@ -355,24 +346,18 @@ def do_install(reqs): # 3) Install: Use our well-populated cache to do the installations. # --use-wheel is somewhat redundant here, but it means we get an error if we have a bad version of pip/setuptools. install_opts = ('--upgrade', '--use-wheel',) + cache_opts - if venv_uses_system_site_packages(): - # In the worst case, a --system-site-packages venv will be missing packages because it was built in an - # environment which satisfied requirements at the system level, then be relocated to a system that doesn't have - # the requirement, breaking it. This has bitten us before. --ignore-installed avoids the issue. - print('Detected --system-site-packages; using --ignore-installed. This makes things slower.') - install_opts += ('--ignore-installed',) recently_installed = pip_install(install_opts + requirements_as_options) required_with_deps = trace_requirements( - (req.req for req in required.values()) + (req.req for req in required) ) # TODO-TEST require A==1 then A==2 extraneous = ( - set(previously_installed) - - set(required_with_deps) - - set(recently_installed) + reqnames(previously_installed) - + reqnames(required_with_deps) - + reqnames(recently_installed) ) # 4) Uninstall any extraneous packages.