Merge pull request #11 from Yelp/faster-find-reqs

Faster find reqs

.travis/test.sh

@@ -10,7 +10,7 @@ coverage --version
 py.test --version
 coverage erase
 py.test -n $NCPU \
-    --cov-enable --cov-config=$TOP/.coveragerc --cov-report='' \
+    --cov --cov-config=$TOP/.coveragerc --cov-report='' \
     "$@" $TOP/tests $SITEPACKAGES/${PROJECT}.py
 coverage combine
 coverage report --rcfile=$TOP/.coveragerc --fail-under 95  # FIXME: should be 100

TODO

@@ -2,7 +2,7 @@ Should probably do before it hits users
 
 * currently if you get your arguments backwards, your requirements get deleted, replaced with empty dir. this is bad
 
-* coverage: 50, 99, 198, 209, 211, 213, 218-221, 47->50, 54->57, 97->99, 192->198
+* coverage: 50, 99, 209, 211, 213, 218-221, 47->50, 54->57, 97->99
 
 
 Things that I want to do, but would put me past my deadline:

requirements.d/_lint.txt

@@ -2,5 +2,6 @@
 flake8
 # pylint  # PR merged: https://bitbucket.org/logilab/pylint/pull-request/186/fixed_up_import_test-and-tests/diff
 # pending fresh release: https://bitbucket.org/logilab/pylint/issue/363/please-cut-a-release-131
-hg+https://bitbucket.org/logilab/pylint@58c66aa083777059a2e6b46f6a0545a2f4977097
+# note: pylint is set up such that -e simply doesn't work. Their __init__ is in their top directory.
+hg+https://bitbucket.org/logilab/pylint@58c66aa083777059a2e6b46f6a0545a2f4977097#egg=pylint
 pre-commit

requirements.d/coverage.txt

@@ -1,13 +1,10 @@
 # Minimal set of pacakges to get coverage working
-# NOTE: versions must be specified in dependency DFS post-order or else we get the wrong versions.
+# NOTE: versions must be specified in post-order or else we get the wrong versions.
 
 #coverage  ## pending pull request: https://bitbucket.org/ned/coveragepy/pull-request/42
-hg+https://bitbucket.org/bukzor/coverage.py@__main__-support
+-e hg+https://bitbucket.org/bukzor/coverage.py@__main__-support#egg=coverage
 
 # cov-core ## pending pull requests:
 #       * https://github.com/schlamar/cov-core/pull/6
 #       * https://github.com/schlamar/cov-core/pull/7
-git+git://github.com/bukzor/cov-core.git@master
-
-# pytest-cov ## pending pull request: https://github.com/schlamar/pytest-cov/pull/21
-git+git://github.com/bukzor/pytest-cov.git@enable-coveragerc
+-e git+git://github.com/bukzor/cov-core.git@master#egg=cov-core

requirements.d/dev.txt

@@ -1,5 +1,5 @@
 # minimal set of packages for committers
--e ..
+-e .
 -r test.txt
 -r _lint.txt
 tox

requirements.d/test.txt

@@ -5,3 +5,9 @@
 pytest-xdist
 pytest-timeout
 pytest-rerunfailures
+
+
+# pytest-cov ## pending pull requests:
+#   * https://github.com/schlamar/pytest-cov/pull/21
+#   * https://github.com/schlamar/pytest-cov/pull/27
+-e git+git://github.com/bukzor/pytest-cov.git@master#egg=pytest-cov

tests/functional/simple_test.py

@@ -31,6 +31,7 @@ def test_second_install_faster(tmpdir):
 coverage
 pylint
 pytest
+six==0.9.0
 ''')
 
     from time import time
@@ -67,18 +68,20 @@ def test_arguments_version(capfd):
         errname = 'FileNotFoundError'
     else:
         errname = 'OSError'
-    assert lasterr == errname + ': [Errno 2] No such file or directory', err
+    assert lasterr.startswith(errname + ': [Errno 2] No such file or directory'), err
 
     lines = out.split('\n')
-    assert lines[-3] == ('> virtualenv virtualenv_run --version'), out
+    assert lines[-4] == ('> virtualenv virtualenv_run --version'), out
+    assert lines[-2].startswith('> /'), out
+    assert lines[-2].endswith('venv_update.py --stage2 virtualenv_run requirements.txt --version'), out
 
 
 def test_arguments_system_packages(tmpdir, capfd):
     """Show that we can pass arguments through to virtualenv"""
     tmpdir.chdir()
     get_scenario('trivial')
 
-    venv_update('--system-site-packages')
+    venv_update('--system-site-packages', 'virtualenv_run', 'requirements.txt')
     out, err = capfd.readouterr()  # flush buffers
 
     run('virtualenv_run/bin/python', '-c', '''\

tests/functional/stage2_test.py

@@ -24,6 +24,10 @@ def test_trivial(tmpdir):
     tmpdir.chdir()
     get_scenario('trivial')
 
+    with open('requirements.txt', 'w') as requirements:
+        # An arbitrary small package: mccabe
+        requirements.write('mccabe\nsix==0.9.0')
+
     run('virtualenv', 'myvenv')
     # need this to get coverage. surely there's a better way...
     run(

venv_update.py

@@ -24,6 +24,7 @@
 
 # This script must not rely on anything other than
 #   stdlib>=2.6 and virtualenv>1.11
+from contextlib import contextmanager
 
 
 def parseargs(args):
@@ -57,10 +58,18 @@ def parseargs(args):
     return stage, virtualenv_dir, tuple(requirements), tuple(remaining)
 
 
+def timid_relpath(arg):
+    from os.path import exists, isabs, relpath
+    if isabs(arg) and exists(arg):
+        return relpath(arg)
+    else:
+        return arg
+
+
 def shellescape(args):
     # TODO: unit test
     from pipes import quote
-    return ' '.join(quote(arg) for arg in args)
+    return ' '.join(quote(timid_relpath(arg)) for arg in args)
 
 
 def colorize(cmd):
@@ -80,9 +89,52 @@ def run(cmd):
     check_call(cmd)
 
 
+@contextmanager
+def faster_pip_packagefinder():
+    """Provide a short-circuited search when the requirement is pinned and appears on disk.
+
+    Suggested upstream at: https://github.com/pypa/pip/pull/2114
+    """
+    from pip.index import PackageFinder, DistributionNotFound, HTMLPage
+
+    orig_packagefinder = vars(PackageFinder).copy()
+
+    def find_requirement(self, req, upgrade):
+        if any(op == '==' for op, ver in req.req.specs):
+            # if the version is pinned-down by a ==, do an optimistic search
+            # for a satisfactory package on the local filesystem.
+            try:
+                self.network_allowed = False
+                result = orig_packagefinder['find_requirement'](self, req, upgrade)
+            except DistributionNotFound:
+                result = None
+
+            if result is not None:
+                return result
+
+        # otherwise, do the full network search
+        self.network_allowed = True
+        return orig_packagefinder['find_requirement'](self, req, upgrade)
+
+    def _get_page(self, link, req):
+        if self.network_allowed or link.url.startswith('file:'):
+            return orig_packagefinder['_get_page'](self, link, req)
+        else:
+            return HTMLPage('', 'fake://' + link.url)
+
+    # A poor man's dependency injection: monkeypatch :(
+    # pylint:disable=protected-access
+    PackageFinder.find_requirement = find_requirement
+    PackageFinder._get_page = _get_page
+    try:
+        yield
+    finally:
+        PackageFinder.find_requirement = orig_packagefinder['find_requirement']
+        PackageFinder._get_page = orig_packagefinder['_get_page']
+
+
 def pip(args):
     """Run pip, in-process."""
-    # NOTE: pip *must* be imported here, so that we get the venv's pip
     import pip as pipmodule
     import sys
 
@@ -93,7 +145,9 @@ def pip(args):
     sys.stdout.write('\n')
     sys.stdout.flush()
 
-    result = pipmodule.main(list(args))
+    with faster_pip_packagefinder():
+        result = pipmodule.main(list(args))
+
     if result != 0:
         # pip exited with failure, then we should too
         exit(result)
@@ -176,6 +230,13 @@ def mark_venv_invalid(venv_path, reqs):
         print()
 
 
+def dotpy(filename):
+    if filename.endswith('.pyc'):
+        return filename[:-1]
+    else:
+        return filename
+
+
 def venv_update(stage, venv_path, reqs, venv_args):
     from os.path import join, abspath
     venv_python = abspath(join(venv_path, 'bin', 'python'))
@@ -184,11 +245,7 @@ def venv_update(stage, venv_path, reqs, venv_args):
         # make a fresh venv at the right spot, and use it to perform stage 2
         clean_venv(venv_path, venv_args)
 
-        from os import execv
-        execv(
-            venv_python,
-            (venv_python, __file__, '--stage2', venv_path) + reqs + venv_args
-        )  # never returns
+        run((venv_python, dotpy(__file__), '--stage2', venv_path) + reqs + venv_args)
     elif stage == 2:
         import sys
         assert sys.executable == venv_python, "Executable not in venv: %s != %s" % (sys.executable, venv_python)