Permalink
Browse files

Adopt cachecontrol 0.12.0 with msgpack support (#3515) (#4258)

This change promises a performance improvement of up to 25%:

    # Before
    $ time pip install django --no-compile
    2.34s user 2.49s system 47% cpu 10.106 total

    # With this patch
    $ time pip install django --no-compile
    1.82s user 2.10s system 51% cpu 7.646 total

Both tests have been performed with hot caches.
  • Loading branch information...
1 parent 0552ffe commit 6da392f6892aa692adf997b5440203b84acf7f2a @StephanErb StephanErb committed with dstufft Jan 30, 2017
View
@@ -1,12 +1,15 @@
**9.1.0 (UNRELEASED)**
+* **WARNING** pip 9.1 cache format has changed. Old versions of pip cannot
+ read cache entries written by 9.1 but will handle this gracefully by
+ re-downloading the affected packages (:issue:`3515`).
+
* Use pkg_resources to parse the entry points file to allow names with
colons (:pull:`3901`)
* Return a failing exit status when `pip install`, `pip download`, or
`pip wheel` is called with no requirements. (:issue:`2720`, :pull:`2721`)
-
* Add `--exclude-editable` to ``pip freeze`` to exclude editable packages
from installed package list.
@@ -4,7 +4,7 @@
"""
__author__ = 'Eric Larson'
__email__ = 'eric@ionrock.org'
-__version__ = '0.11.7'
+__version__ = '0.12.0'
from .wrapper import CacheControl
from .adapter import CacheControlAdapter
@@ -1,18 +1,2 @@
-from textwrap import dedent
-
-try:
- from .file_cache import FileCache
-except ImportError:
- notice = dedent('''
- NOTE: In order to use the FileCache you must have
- lockfile installed. You can install it via pip:
- pip install lockfile
- ''')
- print(notice)
-
-
-try:
- import redis
- from .redis_cache import RedisCache
-except ImportError:
- pass
+from .file_cache import FileCache # noqa
+from .redis_cache import RedisCache # noqa
@@ -1,8 +1,6 @@
import hashlib
import os
-
-from pip._vendor.lockfile import LockFile
-from pip._vendor.lockfile.mkdirlockfile import MkdirLockFile
+from textwrap import dedent
from ..cache import BaseCache
from ..controller import CacheController
@@ -55,19 +53,29 @@ def __init__(self, directory, forever=False, filemode=0o0600,
if use_dir_lock is not None and lock_class is not None:
raise ValueError("Cannot use use_dir_lock and lock_class together")
- if use_dir_lock:
- lock_class = MkdirLockFile
-
- if lock_class is None:
- lock_class = LockFile
+ try:
+ from pip._vendor.lockfile import LockFile
+ from pip._vendor.lockfile.mkdirlockfile import MkdirLockFile
+ except ImportError:
+ notice = dedent("""
+ NOTE: In order to use the FileCache you must have
+ lockfile installed. You can install it via pip:
+ pip install lockfile
+ """)
+ raise ImportError(notice)
+ else:
+ if use_dir_lock:
+ lock_class = MkdirLockFile
+
+ elif lock_class is None:
+ lock_class = LockFile
self.directory = directory
self.forever = forever
self.filemode = filemode
self.dirmode = dirmode
self.lock_class = lock_class
-
@staticmethod
def encode(x):
return hashlib.sha224(x.encode()).hexdigest()
@@ -104,7 +112,10 @@ def set(self, key, value):
def delete(self, key):
name = self._fn(key)
if not self.forever:
- os.remove(name)
+ try:
+ os.remove(name)
+ except FileNotFoundError:
+ pass
def url_to_file_path(url, filecache):
@@ -6,11 +6,11 @@
def total_seconds(td):
"""Python 2.6 compatability"""
if hasattr(td, 'total_seconds'):
- return td.total_seconds()
+ return int(td.total_seconds())
ms = td.microseconds
secs = (td.seconds + td.days * 24 * 3600)
- return (ms + secs * 10**6) / 10**6
+ return int((ms + secs * 10**6) / 10**6)
class RedisCache(object):
@@ -3,23 +3,10 @@
import json
import zlib
+from pip._vendor import msgpack
from pip._vendor.requests.structures import CaseInsensitiveDict
-from .compat import HTTPResponse, pickle, text_type
-
-
-def _b64_encode_bytes(b):
- return base64.b64encode(b).decode("ascii")
-
-
-def _b64_encode_str(s):
- return _b64_encode_bytes(s.encode("utf8"))
-
-
-def _b64_encode(s):
- if isinstance(s, text_type):
- return _b64_encode_str(s)
- return _b64_encode_bytes(s)
+from .compat import HTTPResponse, pickle
def _b64_decode_bytes(b):
@@ -52,14 +39,11 @@ def dumps(self, request, response, body=None):
data = {
"response": {
- "body": _b64_encode_bytes(body),
- "headers": dict(
- (_b64_encode(k), _b64_encode(v))
- for k, v in response.headers.items()
- ),
+ "body": body,
+ "headers": dict(response.headers),
"status": response.status,
"version": response.version,
- "reason": _b64_encode_str(response.reason),
+ "reason": response.reason,
"strict": response.strict,
"decode_content": response.decode_content,
},
@@ -73,20 +57,7 @@ def dumps(self, request, response, body=None):
header = header.strip()
data["vary"][header] = request.headers.get(header, None)
- # Encode our Vary headers to ensure they can be serialized as JSON
- data["vary"] = dict(
- (_b64_encode(k), _b64_encode(v) if v is not None else v)
- for k, v in data["vary"].items()
- )
-
- return b",".join([
- b"cc=2",
- zlib.compress(
- json.dumps(
- data, separators=(",", ":"), sort_keys=True,
- ).encode("utf8"),
- ),
- ])
+ return b",".join([b"cc=3", msgpack.dumps(data, use_bin_type=True)])
def loads(self, request, data):
# Short circuit if we've been given an empty set of data
@@ -174,7 +145,7 @@ def _loads_v1(self, request, data):
def _loads_v2(self, request, data):
try:
cached = json.loads(zlib.decompress(data).decode("utf8"))
- except ValueError:
+ except (ValueError, zlib.error):
return
# We need to decode the items that we've base64 encoded
@@ -194,3 +165,11 @@ def _loads_v2(self, request, data):
)
return self.prepare_response(request, cached)
+
+ def _loads_v3(self, request, data):
+ try:
+ cached = msgpack.loads(data, encoding='utf-8')
+ except ValueError:
+ return
+
+ return self.prepare_response(request, cached)
@@ -0,0 +1,54 @@
+# coding: utf-8
+from pip._vendor.msgpack._version import version
+from pip._vendor.msgpack.exceptions import *
+
+from collections import namedtuple
+
+
+class ExtType(namedtuple('ExtType', 'code data')):
+ """ExtType represents ext type in msgpack."""
+ def __new__(cls, code, data):
+ if not isinstance(code, int):
+ raise TypeError("code must be int")
+ if not isinstance(data, bytes):
+ raise TypeError("data must be bytes")
+ if not 0 <= code <= 127:
+ raise ValueError("code must be 0~127")
+ return super(ExtType, cls).__new__(cls, code, data)
+
+
+import os
+if os.environ.get('MSGPACK_PUREPYTHON'):
+ from pip._vendor.msgpack.fallback import Packer, unpack, unpackb, Unpacker
+else:
+ try:
+ from pip._vendor.msgpack._packer import Packer
+ from pip._vendor.msgpack._unpacker import unpack, unpackb, Unpacker
+ except ImportError:
+ from pip._vendor.msgpack.fallback import Packer, unpack, unpackb, Unpacker
+
+
+def pack(o, stream, **kwargs):
+ """
+ Pack object `o` and write it to `stream`
+
+ See :class:`Packer` for options.
+ """
+ packer = Packer(**kwargs)
+ stream.write(packer.pack(o))
+
+
+def packb(o, **kwargs):
+ """
+ Pack object `o` and return packed bytes
+
+ See :class:`Packer` for options.
+ """
+ return Packer(**kwargs).pack(o)
+
+# alias for compatibility to simplejson/marshal/pickle.
+load = unpack
+loads = unpackb
+
+dump = pack
+dumps = packb
@@ -0,0 +1 @@
+version = (0, 4, 8)
@@ -0,0 +1,29 @@
+class UnpackException(Exception):
+ pass
+
+
+class BufferFull(UnpackException):
+ pass
+
+
+class OutOfData(UnpackException):
+ pass
+
+
+class UnpackValueError(UnpackException, ValueError):
+ pass
+
+
+class ExtraData(ValueError):
+ def __init__(self, unpacked, extra):
+ self.unpacked = unpacked
+ self.extra = extra
+
+ def __str__(self):
+ return "unpack(b) received extra data."
+
+class PackException(Exception):
+ pass
+
+class PackValueError(PackException, ValueError):
+ pass
Oops, something went wrong.

0 comments on commit 6da392f

Please sign in to comment.