Merge pull request #1435 from dstufft/use-raw-stream

Use the raw stream to prevent decoding the response

CHANGES.txt

@@ -7,6 +7,8 @@ Changelog
 * Remove the version requirement for setuptools so that it still functions with
   setuptools < 0.8 installed (Pull #1434).
 
+* Don't decode downloaded files that have a ``Content-Encoding`` header.
+
 
 1.5 (2014-01-01)
 ----------------

pip/download.py

@@ -22,7 +22,8 @@
 from pip._vendor import requests
 from pip._vendor.requests.adapters import BaseAdapter
 from pip._vendor.requests.auth import AuthBase, HTTPBasicAuth
-from pip._vendor.requests.exceptions import InvalidURL
+from pip._vendor.requests.compat import IncompleteRead
+from pip._vendor.requests.exceptions import InvalidURL, ChunkedEncodingError
 from pip._vendor.requests.models import Response
 from pip._vendor.requests.structures import CaseInsensitiveDict
 
@@ -420,7 +421,24 @@ def _download_url(resp, link, temp_location):
             logger.notify('Downloading %s' % show_url)
         logger.info('Downloading from URL %s' % link)
 
-        for chunk in resp.iter_content(4096):
+        def resp_read(chunk_size):
+            try:
+                # Special case for urllib3.
+                try:
+                    for chunk in resp.raw.stream(
+                            chunk_size, decode_content=False):
+                        yield chunk
+                except IncompleteRead as e:
+                    raise ChunkedEncodingError(e)
+            except AttributeError:
+                # Standard file-like object.
+                while True:
+                    chunk = resp.raw.read(chunk_size)
+                    if not chunk:
+                        break
+                    yield chunk
+
+        for chunk in resp_read(4096):
             downloaded += len(chunk)
             if show_progress:
                 if not total_length:

tests/unit/test_download.py

@@ -49,14 +49,23 @@ def _write_file(fn, contents):
         fh.write(contents)
 
 
-class MockResponse(object):
+class FakeStream(object):
 
     def __init__(self, contents):
         self._io = BytesIO(contents)
 
-    def iter_content(self, size):
+    def read(self, size, decode_content=None):
+        return self._io.read(size)
+
+    def stream(self, size, decode_content=None):
         yield self._io.read(size)
 
+
+class MockResponse(object):
+
+    def __init__(self, contents):
+        self.raw = FakeStream(contents)
+
     def raise_for_status(self):
         pass