Browse files

Adopt cachecontrol 0.12.0 with msgpack support (#3515) (#4258)

This change promises a performance improvement of up to 25%:

    # Before
    $ time pip install django --no-compile
    2.34s user 2.49s system 47% cpu 10.106 total

    # With this patch
    $ time pip install django --no-compile
    1.82s user 2.10s system 51% cpu 7.646 total

Both tests have been performed with hot caches.
  • Loading branch information...
StephanErb authored and dstufft committed Jan 30, 2017
1 parent 0552ffe commit 6da392f6892aa692adf997b5440203b84acf7f2a
@@ -1,12 +1,15 @@
**9.1.0 (UNRELEASED)**
* **WARNING** pip 9.1 cache format has changed. Old versions of pip cannot
read cache entries written by 9.1 but will handle this gracefully by
re-downloading the affected packages (:issue:`3515`).
* Use pkg_resources to parse the entry points file to allow names with
colons (:pull:`3901`)
* Return a failing exit status when `pip install`, `pip download`, or
`pip wheel` is called with no requirements. (:issue:`2720`, :pull:`2721`)
* Add `--exclude-editable` to ``pip freeze`` to exclude editable packages
from installed package list.
@@ -4,7 +4,7 @@
__author__ = 'Eric Larson'
__email__ = ''
__version__ = '0.11.7'
__version__ = '0.12.0'
from .wrapper import CacheControl
from .adapter import CacheControlAdapter
@@ -1,18 +1,2 @@
from textwrap import dedent
from .file_cache import FileCache
except ImportError:
notice = dedent('''
NOTE: In order to use the FileCache you must have
lockfile installed. You can install it via pip:
pip install lockfile
import redis
from .redis_cache import RedisCache
except ImportError:
from .file_cache import FileCache # noqa
from .redis_cache import RedisCache # noqa
@@ -1,8 +1,6 @@
import hashlib
import os
from pip._vendor.lockfile import LockFile
from pip._vendor.lockfile.mkdirlockfile import MkdirLockFile
from textwrap import dedent
from ..cache import BaseCache
from ..controller import CacheController
@@ -55,19 +53,29 @@ def __init__(self, directory, forever=False, filemode=0o0600,
if use_dir_lock is not None and lock_class is not None:
raise ValueError("Cannot use use_dir_lock and lock_class together")
if use_dir_lock:
lock_class = MkdirLockFile
if lock_class is None:
lock_class = LockFile
from pip._vendor.lockfile import LockFile
from pip._vendor.lockfile.mkdirlockfile import MkdirLockFile
except ImportError:
notice = dedent("""
NOTE: In order to use the FileCache you must have
lockfile installed. You can install it via pip:
pip install lockfile
raise ImportError(notice)
if use_dir_lock:
lock_class = MkdirLockFile
elif lock_class is None:
lock_class = LockFile = directory
self.forever = forever
self.filemode = filemode
self.dirmode = dirmode
self.lock_class = lock_class
def encode(x):
return hashlib.sha224(x.encode()).hexdigest()
@@ -104,7 +112,10 @@ def set(self, key, value):
def delete(self, key):
name = self._fn(key)
if not self.forever:
except FileNotFoundError:
def url_to_file_path(url, filecache):
@@ -6,11 +6,11 @@
def total_seconds(td):
"""Python 2.6 compatability"""
if hasattr(td, 'total_seconds'):
return td.total_seconds()
return int(td.total_seconds())
ms = td.microseconds
secs = (td.seconds + td.days * 24 * 3600)
return (ms + secs * 10**6) / 10**6
return int((ms + secs * 10**6) / 10**6)
class RedisCache(object):
@@ -3,23 +3,10 @@
import json
import zlib
from pip._vendor import msgpack
from pip._vendor.requests.structures import CaseInsensitiveDict
from .compat import HTTPResponse, pickle, text_type
def _b64_encode_bytes(b):
return base64.b64encode(b).decode("ascii")
def _b64_encode_str(s):
return _b64_encode_bytes(s.encode("utf8"))
def _b64_encode(s):
if isinstance(s, text_type):
return _b64_encode_str(s)
return _b64_encode_bytes(s)
from .compat import HTTPResponse, pickle
def _b64_decode_bytes(b):
@@ -52,14 +39,11 @@ def dumps(self, request, response, body=None):
data = {
"response": {
"body": _b64_encode_bytes(body),
"headers": dict(
(_b64_encode(k), _b64_encode(v))
for k, v in response.headers.items()
"body": body,
"headers": dict(response.headers),
"status": response.status,
"version": response.version,
"reason": _b64_encode_str(response.reason),
"reason": response.reason,
"strict": response.strict,
"decode_content": response.decode_content,
@@ -73,20 +57,7 @@ def dumps(self, request, response, body=None):
header = header.strip()
data["vary"][header] = request.headers.get(header, None)
# Encode our Vary headers to ensure they can be serialized as JSON
data["vary"] = dict(
(_b64_encode(k), _b64_encode(v) if v is not None else v)
for k, v in data["vary"].items()
return b",".join([
data, separators=(",", ":"), sort_keys=True,
return b",".join([b"cc=3", msgpack.dumps(data, use_bin_type=True)])
def loads(self, request, data):
# Short circuit if we've been given an empty set of data
@@ -174,7 +145,7 @@ def _loads_v1(self, request, data):
def _loads_v2(self, request, data):
cached = json.loads(zlib.decompress(data).decode("utf8"))
except ValueError:
except (ValueError, zlib.error):
# We need to decode the items that we've base64 encoded
@@ -194,3 +165,11 @@ def _loads_v2(self, request, data):
return self.prepare_response(request, cached)
def _loads_v3(self, request, data):
cached = msgpack.loads(data, encoding='utf-8')
except ValueError:
return self.prepare_response(request, cached)
@@ -0,0 +1,54 @@
# coding: utf-8
from pip._vendor.msgpack._version import version
from pip._vendor.msgpack.exceptions import *
from collections import namedtuple
class ExtType(namedtuple('ExtType', 'code data')):
"""ExtType represents ext type in msgpack."""
def __new__(cls, code, data):
if not isinstance(code, int):
raise TypeError("code must be int")
if not isinstance(data, bytes):
raise TypeError("data must be bytes")
if not 0 <= code <= 127:
raise ValueError("code must be 0~127")
return super(ExtType, cls).__new__(cls, code, data)
import os
if os.environ.get('MSGPACK_PUREPYTHON'):
from pip._vendor.msgpack.fallback import Packer, unpack, unpackb, Unpacker
from pip._vendor.msgpack._packer import Packer
from pip._vendor.msgpack._unpacker import unpack, unpackb, Unpacker
except ImportError:
from pip._vendor.msgpack.fallback import Packer, unpack, unpackb, Unpacker
def pack(o, stream, **kwargs):
Pack object `o` and write it to `stream`
See :class:`Packer` for options.
packer = Packer(**kwargs)
def packb(o, **kwargs):
Pack object `o` and return packed bytes
See :class:`Packer` for options.
return Packer(**kwargs).pack(o)
# alias for compatibility to simplejson/marshal/pickle.
load = unpack
loads = unpackb
dump = pack
dumps = packb
@@ -0,0 +1 @@
version = (0, 4, 8)
@@ -0,0 +1,29 @@
class UnpackException(Exception):
class BufferFull(UnpackException):
class OutOfData(UnpackException):
class UnpackValueError(UnpackException, ValueError):
class ExtraData(ValueError):
def __init__(self, unpacked, extra):
self.unpacked = unpacked
self.extra = extra
def __str__(self):
return "unpack(b) received extra data."
class PackException(Exception):
class PackValueError(PackException, ValueError):
Oops, something went wrong.

0 comments on commit 6da392f

Please sign in to comment.