Merge pull request #63 from jaraco/feature/cached-installs

Add support for cached installs

CHANGES.rst

@@ -1,3 +1,8 @@
+v9.3.0
+======
+
+#52: ``pip-run`` now honors a ``PIP_RUN_MODE``.
+
 v9.2.1
 ======
 

README.rst

@@ -289,6 +289,22 @@ How Does It Work
 For specifics, see `pip_run.run()
 <https://github.com/jaraco/pip-run/blob/master/pip_run/__init__.py#L9-L16>`_.
 
+
+Environment Persistence
+=======================
+
+``pip-run`` honors the ``PIP_RUN_MODE`` variable. If unset or
+set to ``ephemeral``, depenedncies are installed to an ephemeral
+temporary directory on each invocation (and deleted after).
+Setting this variable to ``persist`` will instead create or re-use
+a directory in the user's cache, only installing the dependencies if
+the directory doesn't already exist. A separate cache is maintained
+for each combination of requirements specified.
+
+``persist`` mode can greatly improve startup performance at the
+expense of staleness and accumulated cruft.
+
+
 Limitations
 ===========
 

conftest.py

@@ -1 +1,54 @@
+import textwrap
+
+import jaraco.path
+import pytest
+
+import pip_run.persist
+
+
 collect_ignore = ['examples']
+
+
+@pytest.fixture
+def reqs_files(tmp_path):
+    """Create a couple of requirements files."""
+    jaraco.path.build(
+        {
+            'reqs1.txt': textwrap.dedent(
+                """
+                abc
+                def
+                """
+            ).lstrip(),
+            'reqs2.txt': textwrap.dedent(
+                """
+                uvw
+                xyz
+                """
+            ).lstrip(),
+        },
+        tmp_path,
+    )
+    return tmp_path.glob('reqs*.txt')
+
+
+@pytest.fixture(scope="session")
+def monkeypatch_session():
+    with pytest.MonkeyPatch.context() as mp:
+        yield mp
+
+
+@pytest.fixture(autouse=True, scope='session')
+def alt_cache_dir(monkeypatch_session, tmp_path_factory):
+    alt_cache = tmp_path_factory.mktemp('cache')
+
+    class Paths:
+        user_cache = alt_cache
+        user_cache_dir = alt_cache
+
+    monkeypatch_session.setattr(pip_run.persist, 'paths', Paths)
+
+
+@pytest.fixture(params=['persist', 'ephemeral'])
+def run_mode(monkeypatch, request):
+    monkeypatch.setenv('PIP_RUN_MODE', request.param)

pip_run/_py38compat.py

@@ -0,0 +1,6 @@
+import sys
+import platform
+
+subprocess_path = (
+    str if sys.version_info < (3, 9) and platform.system() == 'Windows' else lambda x: x
+)

pip_run/deps.py

@@ -2,10 +2,12 @@
 import sys
 import contextlib
 import subprocess
-import tempfile
-import shutil
 import itertools
 import functools
+import argparse
+import pathlib
+import types
+import importlib
 
 import packaging.requirements
 
@@ -14,43 +16,63 @@
 except ImportError:
     import importlib_metadata as metadata  # type: ignore
 
+from ._py38compat import subprocess_path as sp
 
-def _installable(args):
-    """
-    Return True only if the args to pip install
-    indicate something to install.
-
-    >>> _installable(['inflect'])
-    True
-    >>> _installable(['-q'])
-    False
-    >>> _installable(['-q', 'inflect'])
-    True
-    >>> _installable(['-rfoo.txt'])
-    True
-    >>> _installable(['projects/inflect'])
-    True
-    >>> _installable(['~/projects/inflect'])
-    True
-    """
-    return any(
-        not arg.startswith('-')
-        or arg.startswith('-r')
-        or arg.startswith('--requirement')
-        for arg in args
+
+class Install(types.SimpleNamespace):
+
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        '-r',
+        '--requirement',
+        action='append',
+        type=pathlib.Path,
+        default=[],
     )
+    parser.add_argument('package', nargs='*')
+
+    @classmethod
+    def parse(cls, args):
+        parsed, unused = cls.parser.parse_known_args(args)
+        return cls(**vars(parsed))
+
+    def __bool__(self):
+        """
+        Return True only if the args to pip install
+        indicate something to install.
+
+        >>> bool(Install.parse(['inflect']))
+        True
+        >>> bool(Install.parse(['-q']))
+        False
+        >>> bool(Install.parse(['-q', 'inflect']))
+        True
+        >>> bool(Install.parse(['-rfoo.txt']))
+        True
+        >>> bool(Install.parse(['projects/inflect']))
+        True
+        >>> bool(Install.parse(['~/projects/inflect']))
+        True
+        """
+        return bool(self.requirement or self.package)
+
+
+def target_mod():
+    mode = os.environ.get('PIP_RUN_MODE', 'ephemeral')
+    return importlib.import_module(f'.{mode}', package=__package__)
+
+
+def empty(path):
+    return not bool(list(path.iterdir()))
 
 
 @contextlib.contextmanager
 def load(*args):
-    target = tempfile.mkdtemp(prefix='pip-run-')
-    cmd = (sys.executable, '-m', 'pip', 'install', '-t', target) + args
-    env = dict(os.environ, PIP_QUIET="1")
-    _installable(args) and subprocess.check_call(cmd, env=env)
-    try:
-        yield target
-    finally:
-        shutil.rmtree(target)
+    with target_mod().context(args) as target:
+        cmd = (sys.executable, '-m', 'pip', 'install', '-t', sp(target)) + args
+        env = dict(os.environ, PIP_QUIET="1")
+        Install.parse(args) and empty(target) and subprocess.check_call(cmd, env=env)
+        yield str(target)
 
 
 @contextlib.contextmanager

pip_run/ephemeral.py

@@ -0,0 +1,13 @@
+import contextlib
+import tempfile
+import shutil
+import pathlib
+
+
+@contextlib.contextmanager
+def context(args):
+    target = pathlib.Path(tempfile.mkdtemp(prefix='pip-run-'))
+    try:
+        yield target
+    finally:
+        shutil.rmtree(target)

pip_run/persist.py

@@ -0,0 +1,54 @@
+import hashlib
+import contextlib
+
+import app_paths
+
+from . import deps
+
+
+paths = app_paths.AppPaths.get_paths(appname='pip run', appauthor=False)
+
+
+class Hash:
+    """
+    Hash class with support for unicode text.
+    """
+
+    def __init__(self, name):
+        self._hash = hashlib.new(name)
+
+    def update(self, text):
+        self._hash.update(text.encode('utf-8'))
+
+    def hexdigest(self):
+        return self._hash.hexdigest()
+
+
+def cache_key(args):
+    """
+    Generate a cache key representing the packages to be installed.
+
+    >>> reqs1, reqs2 = getfixture('reqs_files')
+    >>> cache_key(['-r', str(reqs1), '--requirement', str(reqs2), 'requests'])
+    '88d9f8a3a4009c1f685a7a724519bd5187e1227d72be6bc7f20a4a02f36d14b3'
+
+    The key should be insensitive to order.
+
+    >>> cache_key(['--requirement', str(reqs2), 'requests', '-r', str(reqs1)])
+    '88d9f8a3a4009c1f685a7a724519bd5187e1227d72be6bc7f20a4a02f36d14b3'
+
+    >>> cache_key(['--foo', '-q'])
+    'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855'
+    """
+    parsed = deps.Install.parse(args)
+    hash = Hash('sha256')
+    for req in sorted(parsed.package):
+        hash.update(req + '\n')
+    for file in sorted(parsed.requirement):
+        hash.update('req:\n' + file.read_text())
+    return hash.hexdigest()
+
+
+@contextlib.contextmanager
+def context(args):
+    yield paths.user_cache.joinpath(cache_key(args))

setup.cfg

@@ -26,6 +26,7 @@ install_requires =
 	more_itertools >= 8.3
 	jaraco.context
 	jaraco.text
+	app_paths
 
 [options.packages.find]
 exclude =

tests/test_deps.py

@@ -1,6 +1,8 @@
 import copy
 
-from pip_run import deps
+import pytest
+
+import pip_run.deps as deps
 
 
 class TestInstallCheck:
@@ -29,6 +31,7 @@ def test_not_installed_args(self):
         assert list(filtered) == expected
 
 
+@pytest.mark.usefixtures('run_mode')
 class TestLoad:
     def test_no_args_passes(self):
         """
@@ -45,3 +48,11 @@ def test_only_options_passes(self):
         """
         with deps.load('-q'):
             pass
+
+
+@pytest.mark.usefixtures('run_mode')
+def test_target_module_context():
+    """Verify a target exists or can be created."""
+    mod = deps.target_mod()
+    with mod.context([]) as target:
+        target.mkdir(exist_ok=True)