backport of #424 (was: deflate'd response: use RFC compliant decompression by default)

CHANGES.txt

@@ -1,3 +1,16 @@
+Unreleased
+----------
+
+Bugfix
+~~~~~~
+
+- Decoding deflate-encoded responses now supports data which is packed in
+  a zlib container as it is supposed to be. The old, non-standard behaviour
+  is still supported.
+
+  See https://github.com/Pylons/webob/pull/426
+
+
 1.8.6 (2020-01-21)
 ------------------
 

src/webob/response.py

@@ -1249,8 +1249,15 @@ def decode_content(self):
             self.content_encoding = None
             gzip_f.close()
         else:
-            # Weird feature: http://bugs.python.org/issue5784
-            self.body = zlib.decompress(self.body, -15)
+            try:
+                # RFC7230 section 4.2.2 specifies that the body should be wrapped
+                # inside a ZLIB (RFC1950) container ...
+                self.body = zlib.decompress(self.body)
+            except zlib.error:
+                # ... but there are nonconformant implementations around which send
+                # the data without the ZLIB container, so we use maximum window size
+                # decompression without header check (the - sign)
+                self.body = zlib.decompress(self.body, -15)
             self.content_encoding = None
 
     def md5_etag(self, body=None, set_content_md5=False):

tests/test_response.py

@@ -381,6 +381,18 @@ def test_decode_content_with_deflate():
     assert res.body == body
     assert res.content_encoding is None
 
+def test_decode_content_with_deflate_and_zlib_header():
+    res = Response()
+    body = b"Hey Hey Hey"
+    # don't chop off the zlib container
+    # https://tools.ietf.org/html/rfc7230#section-4.2.2 says
+    # that chopping it exists but is non-conformant
+    res.body = zlib.compress(body)
+    res.content_encoding = "deflate"
+    res.decode_content()
+    assert res.body == body
+    assert res.content_encoding is None
+
 def test_content_length():
     r0 = Response('x' * 10, content_length=10)