Merge pull request #2595 from msabramo/updated_cachecontrol_to_0.11.2

Update cachecontrol from 0.11.1 to 0.11.2

CHANGES.txt

@@ -35,6 +35,9 @@
 
 * Don't follow symlinks when uninstalling files (:pull:`2552`)
 
+* Upgrade the bundled copy of cachecontrol from 0.11.1 to 0.11.2.
+  Fixes :issue:`2481` (:pull:`2595`)
+
 
 **6.0.8 (2015-02-04)**
 

pip/_vendor/cachecontrol/__init__.py

@@ -4,7 +4,7 @@
 """
 __author__ = 'Eric Larson'
 __email__ = 'eric@ionrock.org'
-__version__ = '0.11.1'
+__version__ = '0.11.2'
 
 from .wrapper import CacheControl
 from .adapter import CacheControlAdapter

pip/_vendor/cachecontrol/adapter.py

@@ -35,10 +35,13 @@ def send(self, request, **kw):
         if request.method == 'GET':
             cached_response = self.controller.cached_request(request)
             if cached_response:
-                return self.build_response(request, cached_response, from_cache=True)
+                return self.build_response(request, cached_response,
+                                           from_cache=True)
 
             # check for etags and add headers if appropriate
-            request.headers.update(self.controller.conditional_headers(request))
+            request.headers.update(
+                self.controller.conditional_headers(request)
+            )
 
         resp = super(CacheControlAdapter, self).send(request, **kw)
 
@@ -74,6 +77,10 @@ def build_response(self, request, response, from_cache=False):
                 response.release_conn()
 
                 response = cached_response
+
+            # We always cache the 301 responses
+            elif response.status == 301:
+                self.controller.cache_response(request, response)
             else:
                 # Check for any heuristics that might update headers
                 # before trying to cache.

pip/_vendor/cachecontrol/caches/file_cache.py

@@ -4,6 +4,7 @@
 from pip._vendor.lockfile import FileLock
 
 from ..cache import BaseCache
+from ..controller import CacheController
 
 
 def _secure_open_write(filename, fmode):
@@ -59,6 +60,8 @@ def encode(x):
         return hashlib.sha224(x.encode()).hexdigest()
 
     def _fn(self, name):
+        # NOTE: This method should not change as some may depend on it.
+        #       See: https://github.com/ionrock/cachecontrol/issues/63
         hashed = self.encode(name)
         parts = list(hashed[:5]) + [hashed]
         return os.path.join(self.directory, *parts)
@@ -89,3 +92,12 @@ def delete(self, key):
         name = self._fn(key)
         if not self.forever:
             os.remove(name)
+
+
+def url_to_file_path(url, filecache):
+    """Return the file cache path based on the URL.
+
+    This does not ensure the file exists!
+    """
+    key = CacheController.cache_url(url)
+    return filecache._fn(key)

pip/_vendor/cachecontrol/compat.py

@@ -10,7 +10,5 @@
     import pickle
 
 
-# Handle the case where the requests has been patched to not have urllib3
-# bundled as part of it's source.
 from pip._vendor.requests.packages.urllib3.response import HTTPResponse
 from pip._vendor.requests.packages.urllib3.util import is_fp_closed

pip/_vendor/cachecontrol/controller.py

@@ -32,7 +32,8 @@ def __init__(self, cache=None, cache_etags=True, serializer=None):
         self.cache_etags = cache_etags
         self.serializer = serializer or Serializer()
 
-    def _urlnorm(self, uri):
+    @classmethod
+    def _urlnorm(cls, uri):
         """Normalize the URL to create a safe key for the cache"""
         (scheme, authority, path, query, fragment) = parse_uri(uri)
         if not scheme or not authority:
@@ -51,8 +52,9 @@ def _urlnorm(self, uri):
 
         return defrag_uri
 
-    def cache_url(self, uri):
-        return self._urlnorm(uri)
+    @classmethod
+    def cache_url(cls, uri):
+        return cls._urlnorm(uri)
 
     def parse_cache_control(self, headers):
         """
@@ -103,7 +105,24 @@ def cached_request(self, request):
         if not resp:
             return False
 
+        # If we have a cached 301, return it immediately. We don't
+        # need to test our response for other headers b/c it is
+        # intrinsically "cacheable" as it is Permanent.
+        # See:
+        #   https://tools.ietf.org/html/rfc7231#section-6.4.2
+        #
+        # Client can try to refresh the value by repeating the request
+        # with cache busting headers as usual (ie no-cache).
+        if resp.status == 301:
+            return resp
+
         headers = CaseInsensitiveDict(resp.headers)
+        if not headers or 'date' not in headers:
+            # With date or etag, the cached response can never be used
+            # and should be deleted.
+            if 'etag' not in headers:
+                self.cache.delete(cache_url)
+            return False
 
         now = time.time()
         date = calendar.timegm(
@@ -182,8 +201,8 @@ def cache_response(self, request, response, body=None):
         This assumes a requests Response object.
         """
         # From httplib2: Don't cache 206's since we aren't going to
-        # handle byte range requests
-        if response.status not in [200, 203]:
+        #                handle byte range requests
+        if response.status not in [200, 203, 300, 301]:
             return
 
         response_headers = CaseInsensitiveDict(response.headers)
@@ -205,6 +224,14 @@ def cache_response(self, request, response, body=None):
                 self.serializer.dumps(request, response, body=body),
             )
 
+        # Add to the cache any 301s. We do this before looking that
+        # the Date headers.
+        elif response.status == 301:
+            self.cache.set(
+                cache_url,
+                self.serializer.dumps(request, response)
+            )
+
         # Add to the cache if the response headers demand it. If there
         # is no date header then we can't do anything about expiring
         # the cache.
@@ -235,7 +262,10 @@ def update_cached_response(self, request, response):
         """
         cache_url = self.cache_url(request.url)
 
-        cached_response = self.serializer.loads(request, self.cache.get(cache_url))
+        cached_response = self.serializer.loads(
+            request,
+            self.cache.get(cache_url)
+        )
 
         if not cached_response:
             # we didn't have a cached response

pip/_vendor/cachecontrol/heuristics.py

@@ -1,9 +1,12 @@
 import calendar
+import time
 
-from email.utils import formatdate, parsedate
+from email.utils import formatdate, parsedate, parsedate_tz
 
 from datetime import datetime, timedelta
 
+TIME_FMT = "%a, %d %b %Y %H:%M:%S GMT"
+
 
 def expire_after(delta, date=None):
     date = date or datetime.now()
@@ -18,7 +21,8 @@ class BaseHeuristic(object):
 
     def warning(self, response):
         """
-        Return a valid 1xx warning header value describing the cache adjustments.
+        Return a valid 1xx warning header value describing the cache
+        adjustments.
 
         The response is provided too allow warnings like 113
         http://tools.ietf.org/html/rfc7234#section-5.5.4 where we need
@@ -36,9 +40,10 @@ def update_headers(self, response):
         return {}
 
     def apply(self, response):
-        warning_header = {'warning': self.warning(response)}
+        warning_header_value = self.warning(response)
         response.headers.update(self.update_headers(response))
-        response.headers.update(warning_header)
+        if warning_header_value is not None:
+            response.headers.update({'Warning': warning_header_value})
         return response
 
 
@@ -77,3 +82,53 @@ def update_headers(self, response):
     def warning(self, response):
         tmpl = '110 - Automatically cached for %s. Response might be stale'
         return tmpl % self.delta
+
+
+class LastModified(BaseHeuristic):
+    """
+    If there is no Expires header already, fall back on Last-Modified
+    using the heuristic from
+    http://tools.ietf.org/html/rfc7234#section-4.2.2
+    to calculate a reasonable value.
+
+    Firefox also does something like this per
+    https://developer.mozilla.org/en-US/docs/Web/HTTP/Caching_FAQ
+    http://lxr.mozilla.org/mozilla-release/source/netwerk/protocol/http/nsHttpResponseHead.cpp#397
+    Unlike mozilla we limit this to 24-hr.
+    """
+    cacheable_by_default_statuses = set([
+        200, 203, 204, 206, 300, 301, 404, 405, 410, 414, 501
+    ])
+
+    def update_headers(self, resp):
+        headers = resp.headers
+
+        if 'expires' in headers:
+            return {}
+
+        if 'cache-control' in headers and headers['cache-control'] != 'public':
+            return {}
+
+        if resp.status not in self.cacheable_by_default_statuses:
+            return {}
+
+        if 'date' not in headers or 'last-modified' not in headers:
+            return {}
+
+        date = calendar.timegm(parsedate_tz(headers['date']))
+        last_modified = parsedate(headers['last-modified'])
+        if date is None or last_modified is None:
+            return {}
+
+        now = time.time()
+        current_age = max(0, now - date)
+        delta = date - calendar.timegm(last_modified)
+        freshness_lifetime = max(0, min(delta / 10, 24 * 3600))
+        if freshness_lifetime <= current_age:
+            return {}
+
+        expires = date + freshness_lifetime
+        return {'expires': time.strftime(TIME_FMT, time.gmtime(expires))}
+
+    def warning(self, resp):
+        return None

pip/_vendor/cachecontrol/serialize.py

@@ -126,7 +126,19 @@ def prepare_response(self, request, cached):
             if request.headers.get(header, None) != value:
                 return
 
-        body = io.BytesIO(cached["response"].pop("body"))
+        body_raw = cached["response"].pop("body")
+
+        try:
+            body = io.BytesIO(body_raw)
+        except TypeError:
+            # This can happen if cachecontrol serialized to v1 format (pickle)
+            # using Python 2. A Python 2 str(byte string) will be unpickled as
+            # a Python 3 str (unicode string), which will cause the above to
+            # fail with:
+            #
+            #     TypeError: 'str' does not support the buffer interface
+            body = io.BytesIO(body_raw.encode('utf8'))
+
         return HTTPResponse(
             body=body,
             preload_content=False,

pip/_vendor/vendor.txt

@@ -3,7 +3,7 @@ html5lib==0.999  # See: https://github.com/html5lib/html5lib-python/issues/161
 six==1.9.0
 colorama==0.3.3
 requests==2.6.0
-CacheControl==0.11.1
+CacheControl==0.11.2
 lockfile==0.10.2
 progress==1.2
 ipaddress==1.0.7  # Only needed on 2.6, 2.7, and 3.2