Place Content-encoding header into response's flags is desired

conftest.py

@@ -21,7 +21,7 @@ def _py_files(folder):
     *_py_files("tests/CrawlerRunner"),
 ]
 
-with open('tests/ignores.txt') as reader:
+with open("tests/ignores.txt") as reader:
     for line in reader:
         file_path = line.strip()
         if file_path and file_path[0] != '#':

scrapy/downloadermiddlewares/httpcompression.py

@@ -27,53 +27,79 @@
 class HttpCompressionMiddleware:
     """This middleware allows compressed (gzip, deflate) traffic to be
     sent/received from web sites"""
-    def __init__(self, stats=None):
+    def __init__(self, stats=None, settings=None):
         self.stats = stats
+        if not stats:
+            warnings.warn(
+                "HttpCompressionMiddleware now accepts a 'stats' parameter which should be specified.",
+                ScrapyDeprecationWarning,
+            )
+        if settings:
+            self.keep_encoding_header = settings.getbool('COMPRESSION_KEEP_ENCODING_HEADER')
+            if not self.keep_encoding_header:
+                warnings.warn(
+                    "COMPRESSION_KEEP_ENCODING_HEADER should be set to True in settings.",
+                    ScrapyDeprecationWarning,
+                )
+        else:
+            self.keep_encoding_header = False
+            warnings.warn(
+                "HttpCompressionMiddleware now accepts a 'settings' parameter which should be specified."
+                "Example: {'COMPRESSION_KEEP_ENCODING_HEADER': True}",
+                ScrapyDeprecationWarning,
+            )
 
     @classmethod
     def from_crawler(cls, crawler):
         if not crawler.settings.getbool('COMPRESSION_ENABLED'):
             raise NotConfigured
         try:
-            return cls(stats=crawler.stats)
+            return cls(stats=crawler.stats, settings=crawler.settings)
         except TypeError:
             warnings.warn(
                 "HttpCompressionMiddleware subclasses must either modify "
-                "their '__init__' method to support a 'stats' parameter or "
+                "their '__init__' method to support a 'stats' and 'settings' parameters or "
                 "reimplement the 'from_crawler' method.",
                 ScrapyDeprecationWarning,
             )
             result = cls()
             result.stats = crawler.stats
+            result.keep_encoding_header = False
             return result
 
     def process_request(self, request, spider):
         request.headers.setdefault('Accept-Encoding',
                                    b", ".join(ACCEPTED_ENCODINGS))
 
     def process_response(self, request, response, spider):
-
         if request.method == 'HEAD':
             return response
-        if isinstance(response, Response):
-            content_encoding = response.headers.getlist('Content-Encoding')
-            if content_encoding:
-                encoding = content_encoding.pop()
-                decoded_body = self._decode(response.body, encoding.lower())
-                if self.stats:
-                    self.stats.inc_value('httpcompression/response_bytes', len(decoded_body), spider=spider)
-                    self.stats.inc_value('httpcompression/response_count', spider=spider)
-                respcls = responsetypes.from_args(
-                    headers=response.headers, url=response.url, body=decoded_body
-                )
-                kwargs = dict(cls=respcls, body=decoded_body)
-                if issubclass(respcls, TextResponse):
-                    # force recalculating the encoding until we make sure the
-                    # responsetypes guessing is reliable
-                    kwargs['encoding'] = None
-                response = response.replace(**kwargs)
-                if not content_encoding:
-                    del response.headers['Content-Encoding']
+
+        if b'decoded' in response.flags:
+            return response
+
+        content_encoding = response.headers.getlist('Content-Encoding')
+        if not content_encoding:
+            return response
+
+        encoding = content_encoding.pop()
+        decoded_body = self._decode(response.body, encoding.lower())
+        if self.stats:
+            self.stats.inc_value('httpcompression/response_bytes', len(decoded_body), spider=spider)
+            self.stats.inc_value('httpcompression/response_count', spider=spider)
+        respcls = responsetypes.from_args(
+            headers=response.headers, url=response.url, body=decoded_body
+        )
+        kwargs = dict(cls=respcls, body=decoded_body)
+        if issubclass(respcls, TextResponse):
+            # force recalculating the encoding until we make sure the
+            # responsetypes guessing is reliable
+            kwargs['encoding'] = None
+
+        kwargs['flags'] = response.flags + [b'decoded']
+        response = response.replace(**kwargs)
+        if not self.keep_encoding_header and not content_encoding:
+            del response.headers['Content-Encoding']
 
         return response
 

scrapy/settings/default_settings.py

@@ -37,6 +37,7 @@
 COMMANDS_MODULE = ''
 
 COMPRESSION_ENABLED = True
+COMPRESSION_KEEP_ENCODING_HEADER = False
 
 CONCURRENT_ITEMS = 100
 

scrapy/templates/project/module/settings.py.tmpl

@@ -86,3 +86,6 @@ ROBOTSTXT_OBEY = True
 #HTTPCACHE_DIR = 'httpcache'
 #HTTPCACHE_IGNORE_HTTP_CODES = []
 #HTTPCACHE_STORAGE = 'scrapy.extensions.httpcache.FilesystemCacheStorage'
+
+# Keep the original Content-Encoding header
+COMPRESSION_KEEP_ENCODING_HEADER = True

tests/test_downloadermiddleware_httpcompression.py

@@ -14,7 +14,6 @@
 from tests import tests_datadir
 from w3lib.encoding import resolve_encoding
 
-
 SAMPLEDIR = join(tests_datadir, 'compressed')
 
 FORMAT = {
@@ -167,6 +166,7 @@ def test_process_response_rawdeflate(self):
         assert 'Content-Encoding' not in newresponse.headers
         self.assertStatsEqual('httpcompression/response_count', 1)
         self.assertStatsEqual('httpcompression/response_bytes', 74840)
+        self.assertIn(b'decoded', newresponse.flags)
 
     def test_process_response_zlibdelate(self):
         response = self._getresponse('zlibdeflate')
@@ -179,6 +179,7 @@ def test_process_response_zlibdelate(self):
         assert 'Content-Encoding' not in newresponse.headers
         self.assertStatsEqual('httpcompression/response_count', 1)
         self.assertStatsEqual('httpcompression/response_bytes', 74840)
+        self.assertIn(b'decoded', newresponse.flags)
 
     def test_process_response_plain(self):
         response = Response('http://scrapytest.org', body=b'<!DOCTYPE...')
@@ -198,6 +199,7 @@ def test_multipleencodings(self):
         newresponse = self.mw.process_response(request, response, self.spider)
         assert newresponse is not response
         self.assertEqual(newresponse.headers.getlist('Content-Encoding'), [b'uuencode'])
+        self.assertIn(b'decoded', newresponse.flags)
 
     def test_process_response_encoding_inside_body(self):
         headers = {
@@ -219,6 +221,7 @@ def test_process_response_encoding_inside_body(self):
         self.assertEqual(newresponse.encoding, resolve_encoding('gb2312'))
         self.assertStatsEqual('httpcompression/response_count', 1)
         self.assertStatsEqual('httpcompression/response_bytes', 104)
+        self.assertIn(b'decoded', newresponse.flags)
 
     def test_process_response_force_recalculate_encoding(self):
         headers = {
@@ -240,6 +243,7 @@ def test_process_response_force_recalculate_encoding(self):
         self.assertEqual(newresponse.encoding, resolve_encoding('gb2312'))
         self.assertStatsEqual('httpcompression/response_count', 1)
         self.assertStatsEqual('httpcompression/response_bytes', 104)
+        self.assertIn(b'decoded', newresponse.flags)
 
     def test_process_response_no_content_type_header(self):
         headers = {
@@ -257,6 +261,7 @@ def test_process_response_no_content_type_header(self):
         self.assertEqual(newresponse.encoding, resolve_encoding('gb2312'))
         self.assertStatsEqual('httpcompression/response_count', 1)
         self.assertStatsEqual('httpcompression/response_bytes', 104)
+        self.assertIn(b'decoded', newresponse.flags)
 
     def test_process_response_gzipped_contenttype(self):
         response = self._getresponse('gzip')
@@ -269,6 +274,7 @@ def test_process_response_gzipped_contenttype(self):
         self.assertNotIn('Content-Encoding', newresponse.headers)
         self.assertStatsEqual('httpcompression/response_count', 1)
         self.assertStatsEqual('httpcompression/response_bytes', 74837)
+        self.assertIn(b'decoded', newresponse.flags)
 
     def test_process_response_gzip_app_octetstream_contenttype(self):
         response = self._getresponse('gzip')
@@ -281,6 +287,7 @@ def test_process_response_gzip_app_octetstream_contenttype(self):
         self.assertNotIn('Content-Encoding', newresponse.headers)
         self.assertStatsEqual('httpcompression/response_count', 1)
         self.assertStatsEqual('httpcompression/response_bytes', 74837)
+        self.assertIn(b'decoded', newresponse.flags)
 
     def test_process_response_gzip_binary_octetstream_contenttype(self):
         response = self._getresponse('x-gzip')
@@ -293,6 +300,7 @@ def test_process_response_gzip_binary_octetstream_contenttype(self):
         self.assertNotIn('Content-Encoding', newresponse.headers)
         self.assertStatsEqual('httpcompression/response_count', 1)
         self.assertStatsEqual('httpcompression/response_bytes', 74837)
+        self.assertIn(b'decoded', newresponse.flags)
 
     def test_process_response_gzipped_gzip_file(self):
         """Test that a gzip Content-Encoded .gz file is gunzipped
@@ -337,6 +345,7 @@ def test_process_response_gzipped_gzip_file(self):
         self.assertEqual(gunzip(newresponse.body), plainbody)
         self.assertStatsEqual('httpcompression/response_count', 1)
         self.assertStatsEqual('httpcompression/response_bytes', 230)
+        self.assertIn(b'decoded', newresponse.flags)
 
     def test_process_response_head_request_no_decode_required(self):
         response = self._getresponse('gzip')
@@ -350,10 +359,42 @@ def test_process_response_head_request_no_decode_required(self):
         self.assertStatsEqual('httpcompression/response_count', None)
         self.assertStatsEqual('httpcompression/response_bytes', None)
 
+    def test_process_response_keeps_content_encoding_header(self):
+        settings = {'COMPRESSION_ENABLED': True,
+                    'COMPRESSION_KEEP_ENCODING_HEADER': True}
+        crawler = get_crawler(Spider, settings)
+        spider = crawler._create_spider('example.com')
+        mw = HttpCompressionMiddleware.from_crawler(crawler)
+        response = self._getresponse('gzip')
+        request = response.request
+
+        self.assertEqual(response.headers['Content-Encoding'], b'gzip')
+        newresponse = mw.process_response(request, response, spider)
+        self.assertIsNot(newresponse, response)
+        self.assertTrue(newresponse.body.startswith(b'<!DOCTYPE'))
+        self.assertIn('Content-Encoding', newresponse.headers)
+        self.assertIn(b'decoded', newresponse.flags)
+
+    def test_process_response_doesnt_keep_content_encoding_header(self):
+        settings = {'COMPRESSION_ENABLED': True,
+                    'COMPRESSION_KEEP_ENCODING_HEADER': False}
+        crawler = get_crawler(Spider, settings)
+        spider = crawler._create_spider('example.com')
+        mw = HttpCompressionMiddleware.from_crawler(crawler)
+        response = self._getresponse('gzip')
+        request = response.request
+
+        self.assertEqual(response.headers['Content-Encoding'], b'gzip')
+        newresponse = mw.process_response(request, response, spider)
+        self.assertIsNot(newresponse, response)
+        self.assertTrue(newresponse.body.startswith(b'<!DOCTYPE'))
+        self.assertNotIn('Content-Encoding', newresponse.headers)
+        self.assertIn(b'decoded', newresponse.flags)
+
 
 class HttpCompressionSubclassTest(TestCase):
 
-    def test_init_missing_stats(self):
+    def test_from_crawler_missing_args(self):
         class HttpCompressionMiddlewareSubclass(HttpCompressionMiddleware):
 
             def __init__(self):
@@ -369,10 +410,28 @@ def __init__(self):
         self.assertEqual(
             messages,
             (
-                (
-                    "HttpCompressionMiddleware subclasses must either modify "
-                    "their '__init__' method to support a 'stats' parameter "
-                    "or reimplement the 'from_crawler' method."
-                ),
-            )
+                "HttpCompressionMiddleware subclasses must either modify "
+                "their '__init__' method to support a 'stats' and 'settings' parameters "
+                "or reimplement the 'from_crawler' method.",
+                "HttpCompressionMiddleware now accepts a 'stats' parameter which should be specified.",
+                "HttpCompressionMiddleware now accepts a 'settings' parameter which should be specified."
+                "Example: {'COMPRESSION_KEEP_ENCODING_HEADER': True}",
+            ),
+        )
+
+    def test_init_missing_args(self):
+        with catch_warnings(record=True) as caught_warnings:
+            self.assertIsNotNone(HttpCompressionMiddleware(stats=None, settings=None))
+        messages = tuple(
+            str(warning.message) for warning in caught_warnings
+            if warning.category is ScrapyDeprecationWarning
+        )
+
+        self.assertEqual(
+            messages,
+            (
+                "HttpCompressionMiddleware now accepts a 'stats' parameter which should be specified.",
+                "HttpCompressionMiddleware now accepts a 'settings' parameter which should be specified."
+                "Example: {'COMPRESSION_KEEP_ENCODING_HEADER': True}",
+            ),
         )