Merge pull request #17 from Yelp/instrument-install.2

Instrument install.2

.travis.yml

@@ -1,27 +1,36 @@
 language: python
-env: # These should match the tox env list
-    # ordered by "most likely to fail in a unique way" first
-    - TOXENV=py34-lint
-#   - TOXENV=pypy3-test
-    - TOXENV=py34-test
-#   - TOXENV=pypy-test
-    - TOXENV=py26-test
-    - TOXENV=py27-test
-    - TOXENV=py27-lint
+python:
+    - "2.6"
+    - "2.7"
+    - "3.4"
+#   - pypy
+#   - pypy3
+env:
+    - TOXENV=lint
+    - TOXENV=test
 matrix:
     # notify a failed build as soon as anything fails
     fast_finish: true
+    exclude:
+        # pylint has dropped 2.6 support -.-
+        - python: "2.6"
+          env: TOXENV=lint
 install: pip install -r requirements.d/travis.txt
 script: tox
-cache:
-    # This doesn't seem to do anything :(
-    directories:
-        - $HOME/.pip
-        - $HOME/.pre-commit
 after_success:
     - find -name '.coverage*' | xargs mv -t .
     - coverage combine
     - coveralls
 after_failure:
     # attempt to show any pip.log on failure
     - find -name pip.log | xargs -r tail -n99999
+
+
+# public caches only supported under sudo:false currently
+#   closest thing to documentation:
+#   http://blog.travis-ci.com/2014-10-07-free-builds-for-students-github-student-developer-pack
+sudo: false
+cache:
+    directories:
+        - $HOME/.pip
+        - $HOME/.pre-commit

.travis/test.sh

@@ -10,4 +10,4 @@ py.test -n $NCPU \
     --cov --cov-config=$TOP/.coveragerc --cov-report='' \
     "$@" $TOP/tests $SITEPACKAGES/${PROJECT}.py
 coverage combine
-coverage report --rcfile=$TOP/.coveragerc --fail-under 97  # FIXME: should be 100
+coverage report --rcfile=$TOP/.coveragerc --fail-under 96  # FIXME: should be 100

TODO

@@ -5,7 +5,7 @@ Should probably do before it hits users
 
 Things that I want to do, but would put me past my deadline:
 
-* coverage: 155, 208, 225, 229-231, 338, 368, 372
+* coverage: 155, 209, 226, 230-232, 303-304, 384, 388
 
 * On ubuntu stock python2.7 I have to rm -rf $VIRTUAL_ENV/local
     to avoid the AssertionError('unexpected third case')

requirements.d/_lint.txt

@@ -1,12 +1,7 @@
 # packages needed for linting, but not for prod
 flake8
-pre-commit
-
 # pylint  # PR merged: https://bitbucket.org/logilab/pylint/pull-request/186/fixed_up_import_test-and-tests/diff
 # pending fresh release: https://bitbucket.org/logilab/pylint/issue/363/please-cut-a-release-131
 # note: pylint is set up such that -e simply doesn't work. Their __init__ is in their top directory.
 hg+https://bitbucket.org/logilab/pylint@58c66aa083777059a2e6b46f6a0545a2f4977097#egg=pylint
-
-# astroid 1.3.0 is broken.
-# see: https://bitbucket.org/logilab/astroid/issue/55/astroid-130-explodes-on-import-pytest
-astroid!=1.3.0
+pre-commit

tests/functional/get_installed_test.py

@@ -1,34 +1,26 @@
 from __future__ import print_function
 from __future__ import unicode_literals
 
-from testing import run, Path
-import venv_update
-
-
-def set_up(tmpdir):
-    tmpdir.chdir()
-
-    run('virtualenv', 'myvenv')
-    # surely there's a better way -.-
-    # NOTE: `pip install TOP` causes an infinite copyfiles loop, under tox
-    Path(venv_update.__file__).copy(tmpdir)
+from testing import run, venv_update_script
 
 
 def get_installed(capfd):
     out, err = capfd.readouterr()  # flush buffers
 
-    run('myvenv/bin/python', '-c', '''\
+    venv_update_script('''\
 import venv_update as v
 for p in sorted(v.pip_get_installed()):
-    print(p)''')
+    print(p)''', venv='myvenv')
 
     out, err = capfd.readouterr()
     assert err == ''
     return out.split()
 
 
 def test_pip_get_installed(tmpdir, capfd):
-    set_up(tmpdir)
+    tmpdir.chdir()
+
+    run('virtualenv', 'myvenv')
     assert get_installed(capfd) == []
     run('myvenv/bin/pip', 'install', 'flake8')
     assert get_installed(capfd) == ['flake8', 'mccabe', 'pep8', 'pyflakes']

tests/functional/install_test.py

@@ -1,22 +1,22 @@
 from __future__ import print_function
 from __future__ import unicode_literals
 
-from testing import run
-
-from functional.get_installed_test import set_up
+from testing import run, venv_update_script
 
 
 def test_pip_install_flake8(tmpdir, capfd):
-    set_up(tmpdir)
+    tmpdir.chdir()
+
+    run('virtualenv', 'myvenv')
     run('myvenv/bin/pip', 'install', 'flake8')
 
     out, err = capfd.readouterr()  # flush buffers
 
-    run('myvenv/bin/python', '-c', '''\
+    venv_update_script('''\
 import json
 from venv_update import pip_install
 print(json.dumps(sorted(pip_install(('flake8',)))))
-''')
+''', venv='myvenv')
 
     out, err = capfd.readouterr()
     assert err == ''

tests/functional/relocation_test.py

@@ -1,4 +1,5 @@
 from __future__ import absolute_import
+from __future__ import print_function
 from __future__ import unicode_literals
 
 import io
@@ -8,6 +9,8 @@
 
 from testing import get_scenario, run, venv_update
 
+PY27 = sys.version_info[:2] == (2, 7)
+
 
 def test_is_relocatable(tmpdir):
     tmpdir.chdir()
@@ -23,15 +26,10 @@ def test_is_relocatable(tmpdir):
 
 def test_is_relocatable_different_python_version(tmpdir):
     tmpdir.chdir()
-    get_scenario('trivial')
-
     with io.open('requirements.txt', 'w') as reqs:
         reqs.write('doge==3.5.0')
 
-    if sys.version_info[:2] == (2, 7):  # pragma: no cover PY27 only
-        python_arg = '--python=python2.6'
-    else:  # pragma: no cover PY26 only
-        python_arg = '--python=python2.7'
+    python_arg = '--python=python' + ('2.7' if not PY27 else '2.6')
 
     venv_update(python_arg)
 

tests/functional/simple_test.py

@@ -3,7 +3,9 @@
 from py._path.local import LocalPath as Path
 import pytest
 
-from testing import get_scenario, run, strip_coverage_warnings, venv_update, TOP
+from testing import (
+    get_scenario, run, strip_coverage_warnings, venv_update, TOP,
+)
 
 from sys import version_info
 PY33 = (version_info >= (3, 3))
@@ -60,10 +62,7 @@ def test_arguments_version(tmpdir, capfd):
     assert excinfo.value.returncode == 1
     out, err = capfd.readouterr()
     lasterr = strip_coverage_warnings(err).rsplit('\n', 2)[-2]
-    if PY33:
-        errname = 'FileNotFoundError'
-    else:
-        errname = 'OSError'
+    errname = 'FileNotFoundError' if PY33 else 'OSError'
     assert lasterr.startswith(errname + ': [Errno 2] No such file or directory'), err
 
     lines = out.split('\n')

tests/testing.py

@@ -34,6 +34,26 @@ def venv_update(*args):
     )
 
 
+def venv_update_script(pyscript, venv='virtualenv_run'):
+    """Run a python script that imports venv_update"""
+    # I wish I didn't need this =/
+    # surely there's a better way -.-
+    # NOTE: `pip install TOP` causes an infinite copyfiles loop, under tox
+    from venv_update import __file__ as venv_update_path, dotpy
+
+    # symlink so that we get coverage, where possible
+    venv_update_path = Path(dotpy(venv_update_path))
+    local_vu = Path(venv_update_path.basename)
+    if local_vu.exists():
+        local_vu.remove()
+    local_vu.mksymlinkto(venv_update_path)
+
+    # write it to a file so we get more-reasonable stack traces
+    testscript = Path('testscript.py')
+    testscript.write(pyscript)
+    run('%s/bin/python' % venv, testscript.strpath)
+
+
 # coverage.py adds some helpful warnings to stderr, with no way to quiet them.
 from re import compile as Regex, MULTILINE
 coverage_warnings_regex = Regex(

tox.ini

@@ -17,7 +17,6 @@ setenv =
     TMPDIR={envtmpdir}
     SITEPACKAGES={envsitepackagesdir}
 commands =
-    python --version
     pip freeze
     test: {toxinidir}/.travis/test.sh {posargs}
     lint: pre-commit run --all {posargs}

venv_update.py

@@ -202,8 +202,9 @@ def pip_install(args):
 
     from pip.commands.install import InstallCommand
     install = InstallCommand()
-    parsed = install.parse_args(list(args))
-    successfully_installed = install.run(*parsed)
+    options, args = install.parse_args(list(args))
+
+    successfully_installed = install.run(options, args)
     if successfully_installed is None:
         return {}
     else:
@@ -281,16 +282,15 @@ def do_install(reqs):
     install_opts = ('--use-wheel',) + cache_opts
     wheel = ('wheel',) + cache_opts
 
-    # Bootstrap the install system; setuptools and pip are alreayd installed, just need wheel
+    # 1) Bootstrap the install system; setuptools and pip are already installed, just need wheel
     pip_install(install_opts + ('wheel',))
 
-    # Caching: Make sure everything we want is downloaded, cached, and has a wheel.
+    # 2) Caching: Make sure everything we want is downloaded, cached, and has a wheel.
     pip(wheel + ('--wheel-dir=' + pip_download_cache, 'wheel') + requirements_as_options)
 
-    # Install: Use our well-populated cache (only) to do the installations.
-    # The --ignore-installed gives more-repeatable behavior in the face of --system-site-packages,
-    #   and brings us closer to a --no-site-packages future
-    recently_installed = pip_install(install_opts + ('--upgrade', '--no-index',) + requirements_as_options)
+    # 3) Install: Use our well-populated cache (only) to do the installations.
+    install_opts += ('--upgrade', '--no-index',)
+    recently_installed = pip_install(install_opts + requirements_as_options)
 
     required_with_deps = trace_requirements(
         (req.req for req in required.values())
@@ -304,7 +304,7 @@ def do_install(reqs):
         set(['wheel'])   # no need to install/uninstall wheel every run
     )
 
-    # Finally, uninstall any extraneous packages
+    # 4) Uninstall any extraneous packages.
     if extraneous:
         pip(('uninstall', '--yes') + tuple(sorted(extraneous)))
 
@@ -337,7 +337,7 @@ def mark_venv_invalid(venv_path, reqs):
 
 
 def dotpy(filename):
-    if filename.endswith('.pyc'):
+    if filename.endswith(('.pyc', '.pyo', '.pyd')):
         return filename[:-1]
     else:
         return filename